{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for deep recurrent Q learning\n",
    "# Paper: Deep Recurrent Q-Learning for Partially Observable MDPs\n",
    "# Paper url: https://arxiv.org/pdf/1507.06527.pdf\n",
    "\n",
    "import gym, random, pickle, os.path, math, glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.autograd as autograd\n",
    "import pdb\n",
    "import os\n",
    "\n",
    "from atari_wrappers import make_atari, wrap_deepmind,LazyFrames\n",
    "from IPython.display import clear_output\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x24c9db45688>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAASKElEQVR4nO3daYwkZ33H8e+vqrtnZmdvH8TetbwgIQ6BbMgGTBxFgDEyBEEUCWRHRChC8RtCTIREcCIF8W5fRAikkEQrjphwOsYEZCHA4hCJhBwb2yHA2iyHY6+vNYuNd3Z2prur/nnRtfaw7OzU9vR0d83z+0ijnqru6Xqerf11Vdfx/BURmNnml026AWY2Hg67WSIcdrNEOOxmiXDYzRLhsJslYl1hl3SNpPsl/VTS+0fVKDMbPQ17nl1SDvwEuBo4AtwJXBcRPx5d88xsVFrr+NtXAD+NiJ8DSPo88BZg1bB3NBOzzK/5xuq0iVYOEmgdLRwViQDIoGiLyId8mxKyHigCyurRNsaKdVa2RZnIOltaeopu78QZU7OesO8BHloxfQR45dn+YJZ5XqmrzvqmarXI91xMcd42Is8oOznkk018ZKLMRdnJWLi4xfKu4drTXgjmHy/Jl0qybkneK0fcUjvlN9bZRSNYZ8vNWGd33vXRVZ9bT9jP9K/3Wx97kq4HrgeYzbfR2rP37O+aZ5Tbt1B2WoMlaPKb9pAgE5GJ/pzobh/2012Ux0SWV+8nTf2WoqlWrrNiTvS2DftOovyVyPrNX2frCfsR4JIV03uBR05/UUQcBA4CbN+2J7qXnr/2O0tTd54gMhEt6M9Db8dwn+4qc4oOZD2R9Sb/IbbZnVpnvXno7hxynUW2adbZesJ+J/B8Sc8FHgauBf50zb+a8C750CJQIbIu5IvZmscSohWU7aj2TqqZauYWofHE4N/+1HqIQXh1Wngjh7ITz64nQTT0v+uZDB32iOhL+kvg60AOfCIifjSylk0RlTE4SFMU7HhAbHl8jf8BgpPnZyxePAh85DF1eypJCzF7NGP+kUDlsx/Ag3UG0YKyxab7cF7Plp2I+Crw1RG1ZWopAhUBBeTHusyu8fqQKPMZTj4HlA92J89wOMMmRAHt47D14S5ZMdi9D4myNcPShVBkg+/lm22NrSvsqYhTBwkFxUy+5qm3kChmmI7ThmYVh70uQZmL3taM7vza++S9rSKyIDI23e6gNZPDXtOpUzlFu9pqr6Fsb66DO9Z8DnsdgmiJsi2WdmcsrXX2UFDMBNEOIgvvzttUcNhrCA0uzijborsdls8vHGBrHId9GKcFPVsSWVdrfjVvnRBZL8j6AQ29Csuay2FfrxJmjmVseSzQGhdptZZLOscLsm6JGnBThW0uDvsIZD2YOV6uGfasFw76tKj2zmIK7r0YF4fdkhQ5FLPZb3xAl63qDMomPVXqsFuSyhb0504Le4dNfVmzwz4C0YLeXLbmrnneHRygo1+iYkyNs98SGgS7tyV79tp4QdERoc17qtRhXy/B8u6gP7fy9rYz6zwN2x6C1qKgOxgMwd/dJ0DB8q6gmP3NdVbMBGVn81716LCvl6DYUlJsqfPajOLx6t7o/ibdfDTBuayzTcRhr0lloL5oL0D/yeG+2LWP+zz7OJ1aZ60T0Hlq2HU2ONuyGdaZw16DyoA+ZCqZ/VU59IglrZNBe7F8JvDehd84Xme/baxhj0wUM0MO8zlBUY2Fd+rW1rXOp59N2RIEhDKiqaP2NECq62wwdsKZjTXs/Tnxy5d2xrnIkYnqWE7ZHvwMQwUs9vLBf7zYlMeApkqK66x/7zrCLukTwJuAoxHxkmrebuALwD7gAeBtEfHkWu9VdmDhUp9zMtso5Vm2pXW27P8K/CPwqRXz3g98MyIOVGWf3g/8zVpvlHUK5i85XmORZjaMrLP6xnTNsEfEdyXtO232W4BXV7/fBHyHGmG/dMsx/uWyT6/1MjMb0l9sObbqc8N+Z39ORDwKEBGPSrqwzh9tlbhydhNfj2g2YVvPcmPPhidP0vWS7pJ01xPH/H3dbFKGDfvjki4CqB6PrvbCiDgYEfsjYv8F5zXvtJvZZjFs2L8CvKP6/R3Al0fTHDPbKGuGXdLngO8BL5B0RNI7gQPA1ZIOM6jPfmBjm2lm61XnaPx1qzx19trLZjZVxnoF3VNlxldOJHarkdkYPVWuvrM+1rAf7W7jnx56zTgXaZaUo93HVn1urGHvFTkP/3rHOBdplpResfoZr/He4rqQo//cOdZFmiVlYUrC3l4MLrx7aZyLNEvKg4ur35Y33i17NPvmf7Opd5Z8+UJ1s0Q47GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZIhx2s0Q47GaJcNjNElFnWKpLJH1b0iFJP5J0QzV/t6TbJR2uHndtfHPNbFh1tux94L0R8SLgCuBdkl7Ms1Vhng98s5o2sym1Ztgj4tGIuLv6/ThwCNjDoCrMTdXLbgL+eIPaaGYjcE7f2asyUC8D7uC0qjDAGavCrCwS0e2dWGdzzWxYtcMuaSvwReA9EfF03b9bWSSi054fpo1mNgK1wi6pzSDon4mIW6vZtavCmNnk1TkaL+DjwKGI+NCKp1wVxqxB6gxLdSXwZ8D/Srq3mve3DKrA3FxViHkQeOuGtNDMRqJORZj/AlarA+uqMGYN4SvozBLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRJRZwy6WUn/Lel/qoowH6zmuyKMWYPU2bIvA6+NiMuAy4FrJF2BK8KYNUqdijAREQvVZLv6CVwRxqxR6o4bn1cjyx4Fbo8IV4Qxa5haYY+IIiIuB/YCr5D0kroLcEUYs+lwTkfjI+Ip4DvANbgijFmj1Dkaf4GkndXvc8DrgPtwRRizRqlTEeYi4CZJOYMPh5sj4jZJ38MVYcwao05FmB8wKNN8+vxjuCKMWWP4CjqzRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRNQOezWc9D2SbqumXRHGrEHOZct+A3BoxbQrwpg1SN0iEXuBPwI+tmK2K8KYNUjdLfuHgfcB5Yp5rghj1iB1xo1/E3A0Ir4/zAJcEcZsOtQZN/5K4M2S3gjMAtslfZqqIkxEPOqKMGbTr04V1xsjYm9E7AOuBb4VEW/HFWHMGmU959kPAFdLOgxcXU2b2ZSqsxv/jIj4DoPCjq4IY9YwvoLOLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBG1RqqR9ABwHCiAfkTsl7Qb+AKwD3gAeFtEPLkxzTSz9TqXLftrIuLyiNhfTbsijFmDrGc33hVhzBqkbtgD+Iak70u6vprnijBmDVJ3dNkrI+IRSRcCt0u6r+4CIuIgcBBg+7Y9MUQbzWwEam3ZI+KR6vEo8CXgFVQVYQBcEcZs+tWp9TYvadup34HXAz/EFWHMGqXObvxzgC9JOvX6z0bE1yTdCdws6Z3Ag8BbN66ZZrZea4Y9In4OXHaG+a4IY9YgvoLOLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBG1wi5pp6RbJN0n6ZCkV0naLel2SYerx10b3VgzG17dLftHgK9FxAsZDFF1CFeEMWuUOqPLbgf+EPg4QER0I+IpXBHGrFHqbNmfBzwBfFLSPZI+Vg0pPfGKMCFR5hllK6NoZxQzOUV7MF3mGZFppMsza7I6Q0m3gJcD746IOyR9hHPYZd/IijD9uZzujpwyh/6cKGZE1gs6C0HWC9qLJa0TBQoXojGrs2U/AhyJiDuq6VsYhH/iFWHKjuhuE8s7M05eIBYvCk5eKJZ3iO62jP5sBt64mwE1wh4RjwEPSXpBNesq4MdMQ0UYQZlD2YJoPfsYuYgMB91shbqFHd8NfEZSB/g58OcMPigmWhGmzEXZEWUHitmgmAsEFB1BKcrcaTc7pVbYI+JeYP8Znpp4RZgQRDb4IQ8iW7FVd9bNnuEr6MwS4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WiMaHXTH4IYBy8PPMtMesMHtG3Vtcp1LeDdrHg3wJ1BfFQk7Whc6vg7w7eN6BNxtodNgHQ1CVRC7yZSjbkPWhvRhk/SBfKifdRLOp0eiwqwzyXlCWg46UBagYfAhk/UClN+tmp6wZ9mo4qi+smPU84O+BT1Xz9wEPAG+LiCdH38TVZb2S1kJAJkKABBGoiMF3+TI82KRZpc4YdPdHxOURcTnwu8Ai8CWmoEjEYMteki8XtJYKWif7tJYK8l5J1i+9ZTdb4VyPxl8F/Cwi/g8XiTBrlHMN+7XA56rfaxWJMLPpUDvs1ciybwb+/VwWsJEVYcysvnPZsr8BuDsiHq+maxWJiIiDEbE/IvZ32vPra62ZDe1cwn4dz+7CwzQUiTCz2urWZ98CXA3cumL2AeBqSYer5w6MvnlmNip1i0QsAuedNu8YU1AkwszqafyNMGZWj8NulgiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyWi0SPVmE27Ms+IDMhEmQsEKgdjMVAGWTG+EZUcdrMNEhK97TndrRlFRyzvFGUH8pPQPhFkPdjyyz7thf5Y2uOwm20UQX82Y3l7RjEHJy8MirmS1kJG+ZTIl2DmeEZ7YTzNGWvYi07G8b0z41yk2cREJpZ2i+4OKGaD/o4CzfXptVqQ5WTLQv2c3tzoMlH8YPXDcGMNe38Ojr1U41yk2eQI+tv6aL5P3i65YMcJtrR7PL00w8KJWXr9jO6uDvni6I6T97+9+nPj3Y3Pg/7OYqyLNJsYBfnWPrNzXTqtPrvnFtnS6pJnJVLQK3IWStHvjDCG+eoH+8Ya9p3zi/zJ7901zkWaTdT21hLb8iXaKtjdWqCtPovlDMeLOXqR86v+PCeL9siW99n5xVWfG2vY97ZPcOB37hznIs0mKmPtr63lCGuUfbe9+jiPYw27EG3l41yk2dQbZSJ0lg+XusNS/bWkH0n6oaTPSZqVtFvS7ZIOV4+7RtZiMxu5NcMuaQ/wV8D+iHgJgw+ia5mCijBmVl/dY/4tYE5SC9gCPIIrwpg1Sp1abw8D/wA8CDwK/DoivoErwpg1Sp3d+F0MtuLPBS4G5iW9ve4CVlaEeeKYz7GbTUqd3fjXAb+IiCciosdg7PjfZ4iKMBec5yPxZpNSJ+wPAldI2iJJDMaKP4Qrwpg1yprn2SPiDkm3AHcDfeAe4CCwFbhZ0jsZfCC8dSMbambrU7cizAeAD5w2exlXhDFrDA9LZZYIh90sEQ67WSIcdrNEKGI8I1sCSHoCOAH8cmwL3Xjn4/5Ms83Unzp9uTQiLjjTE2MNO4CkuyJi/1gXuoHcn+m2mfqz3r54N94sEQ67WSImEfaDE1jmRnJ/pttm6s+6+jL27+xmNhnejTdLxFjDLukaSfdL+qmkRg1jJekSSd+WdKgaj++Gan6jx+KTlEu6R9Jt1XRj+yNpp6RbJN1XradXNbw/Ix37cWxhl5QDHwXeALwYuE7Si8e1/BHoA++NiBcBVwDvqtrf9LH4bmBwy/IpTe7PR4CvRcQLgcsY9KuR/dmQsR8jYiw/wKuAr6+YvhG4cVzL34D+fBm4GrgfuKiadxFw/6Tbdg592Fv9h3ktcFs1r5H9AbYDv6A6DrViflP7swd4CNjN4O7U24DXr6c/49yNP9X4U45U8xpH0j7gZcAdNHssvg8D7wPKFfOa2p/nAU8An6y+lnxM0jwN7U9swNiP4wz7mUavb9ypAElbgS8C74mIpyfdnmFJehNwNCK+P+m2jEgLeDnwzxHxMgaXZTdil/1M1jv245mMM+xHgEtWTO9lMCR1Y0hqMwj6ZyLi1mp2rbH4ptCVwJslPQB8HnitpE/T3P4cAY5ExB3V9C0Mwt/U/qxr7MczGWfY7wSeL+m5kjoMDjZ8ZYzLX5dq/L2PA4ci4kMrnmrkWHwRcWNE7I2IfQzWxbci4u00tz+PAQ9JekE16yrgxzS0P2zE2I9jPujwRuAnwM+Av5v0QZBzbPsfMPja8QPg3urnjcB5DA5yHa4ed0+6rUP07dU8e4Cusf0BLgfuqtbRfwC7Gt6fDwL3AT8E/g2YWU9/fAWdWSJ8BZ1ZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulgiH3SwR/w+Ggo2PieYLqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create and wrap the environment\n",
    "env = make_atari('PongNoFrameskip-v4') # only use in no frameskip environment\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True )\n",
    "n_actions = env.action_space.n\n",
    "state_dim = env.observation_space.shape\n",
    "\n",
    "# env.render()\n",
    "test = env.reset()\n",
    "for i in range(100):\n",
    "    test = env.step(env.action_space.sample())[0]\n",
    "\n",
    "plt.imshow(test._force()[...,0])\n",
    "\n",
    "# plt.imshow(env.render(\"rgb_array\"))\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, in_channels=4, num_actions=5):\n",
    "        \"\"\"\n",
    "        Initialize a deep Q-learning network as described in\n",
    "        https://storage.googleapis.com/deepmind-data/assets/papers/DeepMindNature14236Paper.pdf\n",
    "        Arguments:\n",
    "            in_channels: number of channel of input.\n",
    "                i.e The number of most recent frames stacked together as describe in the paper\n",
    "            num_actions: number of action-value to output, one-to-one correspondence to action in game.\n",
    "        \"\"\"\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc4 = nn.Linear(7 * 7 * 64, 512)\n",
    "        self.fc5 = nn.Linear(512, num_actions)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.contiguous().view(x.size(0), -1)))\n",
    "        return self.fc5(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Nstep_Memory_Buffer(object):\n",
    "    # memory buffer to store episodic memory\n",
    "    def __init__(self, memory_size=1000, n_multi_step = 1, gamma = 0.99):\n",
    "        self.buffer = []\n",
    "        self.memory_size = memory_size\n",
    "        self.n_multi_step = n_multi_step\n",
    "        self.gamma = gamma\n",
    "        self.next_idx = 0\n",
    "        \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        data = (state, action, reward, next_state, done)\n",
    "        if len(self.buffer) <= self.memory_size: # buffer not full\n",
    "            self.buffer.append(data)\n",
    "        else: # buffer is full\n",
    "            self.buffer[self.next_idx] = data\n",
    "        self.next_idx = (self.next_idx + 1) % self.memory_size\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        # sample episodic memory\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            finish = random.randint(self.n_multi_step, self.size() - 1)\n",
    "            begin = finish-self.n_multi_step\n",
    "            sum_reward = 0 # n_step rewards\n",
    "            data = self.buffer[begin:finish]\n",
    "            state = data[0][0]\n",
    "            action = data[0][1]\n",
    "            for j in range(self.n_multi_step):\n",
    "                # compute the n-th reward\n",
    "                sum_reward += (self.gamma**j) * data[j][2]\n",
    "                if data[j][4]:\n",
    "                    # manage end of episode\n",
    "                    states_look_ahead = data[j][3]\n",
    "                    done_look_ahead = True\n",
    "                    break\n",
    "                else:\n",
    "                    states_look_ahead = data[j][3]\n",
    "                    done_look_ahead = False\n",
    "            \n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            rewards.append(sum_reward)\n",
    "            next_states.append(states_look_ahead)\n",
    "            dones.append(done_look_ahead)\n",
    "\n",
    "        return np.concatenate(states), actions, rewards, np.concatenate(next_states), dones\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self.buffer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Nstep_DQNAgent: \n",
    "    def __init__(self, in_channels = 1, action_space = [], USE_CUDA = False, memory_size = 10000, n_multi_step = 16, gamma = 0.99, epsilon  = 1, lr = 1e-4):\n",
    "        self.epsilon = epsilon\n",
    "        self.action_space = action_space\n",
    "        self.n_multi_step = n_multi_step\n",
    "        self.gamma = gamma \n",
    "        self.memory_buffer = Nstep_Memory_Buffer(memory_size, n_multi_step = n_multi_step, gamma = gamma)\n",
    "        self.DQN = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target.load_state_dict(self.DQN.state_dict())\n",
    "\n",
    "\n",
    "        self.USE_CUDA = USE_CUDA\n",
    "        if USE_CUDA:\n",
    "            self.DQN = self.DQN.cuda()\n",
    "            self.DQN_target = self.DQN_target.cuda()\n",
    "        self.optimizer = optim.RMSprop(self.DQN.parameters(),lr=lr, eps=0.001, alpha=0.95)\n",
    "\n",
    "    def observe(self, lazyframe):\n",
    "        # from Lazy frame to tensor\n",
    "        state =  torch.from_numpy(lazyframe._force().transpose(2,0,1)[None]/255).float()\n",
    "        if self.USE_CUDA:\n",
    "            state = state.cuda()\n",
    "        return state\n",
    "\n",
    "    def value(self, state):\n",
    "        q_values = self.DQN(state)\n",
    "        return q_values\n",
    "    \n",
    "    def act(self, state, epsilon = None):\n",
    "        \"\"\"\n",
    "        sample actions with epsilon-greedy policy\n",
    "        recap: with p = epsilon pick random action, else pick action with highest Q(s,a)\n",
    "        \"\"\"\n",
    "        if epsilon is None: epsilon = self.epsilon\n",
    "\n",
    "        q_values = self.value(state).cpu().detach().numpy()\n",
    "        if random.random()<epsilon:\n",
    "            aciton = random.randrange(self.action_space.n)\n",
    "        else:\n",
    "            aciton = q_values.argmax(1)[0]\n",
    "        return aciton\n",
    "    \n",
    "    def compute_td_loss(self, states, actions, rewards, next_states, is_done, gamma=0.99):\n",
    "        \"\"\" Compute td loss using torch operations only. Use the formula above. \"\"\"\n",
    "        actions = torch.tensor(actions).long()    # shape: [batch_size]\n",
    "        rewards = torch.tensor(rewards, dtype =torch.float)  # shape: [batch_size]\n",
    "        is_done = torch.tensor(is_done).bool()  # shape: [batch_size]\n",
    "        \n",
    "        if self.USE_CUDA:\n",
    "            actions = actions.cuda()\n",
    "            rewards = rewards.cuda()\n",
    "            is_done = is_done.cuda()\n",
    "\n",
    "        # get q-values for all actions in current states\n",
    "        predicted_qvalues = self.DQN(states)\n",
    "\n",
    "        # select q-values for chosen actions\n",
    "        predicted_qvalues_for_actions = predicted_qvalues[\n",
    "          range(states.shape[0]), actions\n",
    "        ]\n",
    "\n",
    "        # compute q-values for all actions in next states\n",
    "        predicted_next_qvalues = self.DQN_target(next_states) # YOUR CODE\n",
    "\n",
    "        # compute V*(next_states) using predicted next q-values\n",
    "        next_state_values =  predicted_next_qvalues.max(-1)[0] # YOUR CODE\n",
    "\n",
    "        # compute \"target q-values\" for loss - it's what's inside square parentheses in the above formula.\n",
    "        target_qvalues_for_actions = rewards + (self.gamma**self.n_multi_step) *next_state_values\n",
    "\n",
    "        # at the last state we shall use simplified formula: Q(s,a) = r(s,a) since s' doesn't exist\n",
    "        target_qvalues_for_actions = torch.where(\n",
    "            is_done, rewards, target_qvalues_for_actions)\n",
    "\n",
    "        # mean squared error loss to minimize\n",
    "        #loss = torch.mean((predicted_qvalues_for_actions -\n",
    "        #                   target_qvalues_for_actions.detach()) ** 2)\n",
    "        loss = F.smooth_l1_loss(predicted_qvalues_for_actions, target_qvalues_for_actions.detach())\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def sample_from_buffer(self, batch_size):\n",
    "        # sample episodic memory\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            finish = random.randint(self.n_multi_step, self.memory_buffer.size() - 1)\n",
    "            begin = finish-self.n_multi_step\n",
    "            sum_reward = 0 # n_step rewards\n",
    "            data = self.memory_buffer.buffer[begin:finish]\n",
    "            state = data[0][0] # s0\n",
    "            action = data[0][1] # a0\n",
    "            for j in range(self.n_multi_step):\n",
    "                # compute the n-th reward\n",
    "                sum_reward += (self.gamma**j) * data[j][2] # sum reward\n",
    "                if data[j][4]:\n",
    "                    # manage end of episode\n",
    "                    states_look_ahead = data[j][3] # st\n",
    "                    done_look_ahead = True\n",
    "                    break\n",
    "                else:\n",
    "                    states_look_ahead = data[j][3] # st\n",
    "                    done_look_ahead = False\n",
    "            \n",
    "            states.append(self.observe(state))\n",
    "            actions.append(action)\n",
    "            rewards.append(sum_reward)\n",
    "            next_states.append(self.observe(states_look_ahead))\n",
    "            dones.append(done_look_ahead)\n",
    "\n",
    "        return torch.cat(states), actions, rewards, torch.cat(next_states), dones\n",
    "\n",
    "    def learn_from_experience(self, batch_size):\n",
    "        if self.memory_buffer.size() > batch_size:\n",
    "            states, actions, rewards, next_states, dones = self.sample_from_buffer(batch_size)\n",
    "            td_loss = self.compute_td_loss(states, actions, rewards, next_states, dones)\n",
    "            self.optimizer.zero_grad()\n",
    "            td_loss.backward()\n",
    "            for param in self.DQN.parameters():\n",
    "                param.grad.data.clamp_(-1, 1)\n",
    "\n",
    "            self.optimizer.step()\n",
    "            return(td_loss.item())\n",
    "        else:\n",
    "            return(0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:     0, reward:   nan, loss: 0.000000, epsilon: 1.000000, episode:    0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:  1000, reward:   nan, loss: 0.000000, epsilon: 0.968855, episode:    0\n",
      "frames:  2000, reward: -21.000000, loss: 0.000000, epsilon: 0.938732, episode:    1\n",
      "frames:  3000, reward: -20.000000, loss: 0.000000, epsilon: 0.909596, episode:    2\n",
      "frames:  4000, reward: -20.000000, loss: 0.000000, epsilon: 0.881415, episode:    3\n",
      "frames:  5000, reward: -20.250000, loss: 0.000000, epsilon: 0.854158, episode:    4\n",
      "frames:  6000, reward: -20.500000, loss: 0.000000, epsilon: 0.827794, episode:    6\n",
      "frames:  7000, reward: -20.428571, loss: 0.000000, epsilon: 0.802295, episode:    7\n",
      "frames:  8000, reward: -20.500000, loss: 0.000000, epsilon: 0.777632, episode:    8\n",
      "frames:  9000, reward: -20.555556, loss: 0.000000, epsilon: 0.753777, episode:    9\n",
      "frames: 10000, reward: -20.500000, loss: 0.075068, epsilon: 0.730705, episode:   10\n",
      "frames: 11000, reward: -20.700000, loss: 0.053801, epsilon: 0.708389, episode:   12\n",
      "frames: 12000, reward: -20.800000, loss: 0.079203, epsilon: 0.686804, episode:   13\n",
      "frames: 13000, reward: -20.800000, loss: 0.089163, epsilon: 0.665927, episode:   14\n",
      "frames: 14000, reward: -20.500000, loss: 0.056013, epsilon: 0.645735, episode:   15\n",
      "frames: 15000, reward: -20.500000, loss: 0.000970, epsilon: 0.626204, episode:   16\n",
      "frames: 16000, reward: -20.500000, loss: 0.037479, epsilon: 0.607314, episode:   17\n",
      "frames: 17000, reward: -20.500000, loss: 0.004159, epsilon: 0.589043, episode:   18\n",
      "frames: 18000, reward: -20.400000, loss: 0.099974, epsilon: 0.571371, episode:   19\n",
      "frames: 19000, reward: -20.500000, loss: 0.057437, epsilon: 0.554278, episode:   20\n",
      "frames: 20000, reward: -20.300000, loss: 0.021491, epsilon: 0.537746, episode:   21\n",
      "frames: 21000, reward: -20.300000, loss: 0.028581, epsilon: 0.521756, episode:   22\n",
      "frames: 22000, reward: -20.300000, loss: 0.042032, epsilon: 0.506290, episode:   23\n",
      "frames: 23000, reward: -20.200000, loss: 0.048410, epsilon: 0.491331, episode:   24\n",
      "frames: 24000, reward: -20.500000, loss: 0.041170, epsilon: 0.476863, episode:   25\n",
      "frames: 25000, reward: -20.600000, loss: 0.044129, epsilon: 0.462868, episode:   27\n",
      "frames: 26000, reward: -20.400000, loss: 0.019199, epsilon: 0.449333, episode:   28\n",
      "frames: 27000, reward: -20.400000, loss: 0.086410, epsilon: 0.436241, episode:   29\n",
      "frames: 28000, reward: -20.400000, loss: 0.028228, epsilon: 0.423579, episode:   30\n",
      "frames: 29000, reward: -20.600000, loss: 0.082689, epsilon: 0.411331, episode:   31\n",
      "frames: 30000, reward: -20.500000, loss: 0.069054, epsilon: 0.399485, episode:   33\n",
      "frames: 31000, reward: -20.600000, loss: 0.042428, epsilon: 0.388028, episode:   34\n",
      "frames: 32000, reward: -20.400000, loss: 0.040518, epsilon: 0.376946, episode:   35\n",
      "frames: 33000, reward: -20.400000, loss: 0.059383, epsilon: 0.366228, episode:   36\n",
      "frames: 34000, reward: -20.400000, loss: 0.058692, epsilon: 0.355860, episode:   37\n",
      "frames: 35000, reward: -20.500000, loss: 0.044310, epsilon: 0.345833, episode:   38\n",
      "frames: 36000, reward: -20.500000, loss: 0.056349, epsilon: 0.336135, episode:   39\n",
      "frames: 37000, reward: -20.500000, loss: 0.019648, epsilon: 0.326754, episode:   40\n",
      "frames: 38000, reward: -20.500000, loss: 0.017039, epsilon: 0.317681, episode:   41\n",
      "frames: 39000, reward: -20.500000, loss: 0.029084, epsilon: 0.308905, episode:   42\n",
      "frames: 40000, reward: -20.500000, loss: 0.058250, epsilon: 0.300417, episode:   44\n",
      "frames: 41000, reward: -20.700000, loss: 0.022038, epsilon: 0.292208, episode:   45\n",
      "frames: 42000, reward: -20.700000, loss: 0.022726, epsilon: 0.284267, episode:   46\n",
      "frames: 43000, reward: -20.700000, loss: 0.028613, epsilon: 0.276587, episode:   47\n",
      "frames: 44000, reward: -20.800000, loss: 0.023286, epsilon: 0.269159, episode:   48\n",
      "frames: 45000, reward: -20.900000, loss: 0.020695, epsilon: 0.261974, episode:   50\n",
      "frames: 46000, reward: -20.900000, loss: 0.004817, epsilon: 0.255024, episode:   51\n",
      "frames: 47000, reward: -20.900000, loss: 0.007118, epsilon: 0.248303, episode:   52\n",
      "frames: 48000, reward: -20.900000, loss: 0.013425, epsilon: 0.241802, episode:   53\n",
      "frames: 49000, reward: -20.900000, loss: 0.034531, epsilon: 0.235514, episode:   54\n",
      "frames: 50000, reward: -20.800000, loss: 0.015398, epsilon: 0.229432, episode:   56\n",
      "frames: 51000, reward: -20.700000, loss: 0.005150, epsilon: 0.223549, episode:   57\n",
      "frames: 52000, reward: -20.500000, loss: 0.003733, epsilon: 0.217860, episode:   58\n",
      "frames: 53000, reward: -20.500000, loss: 0.006496, epsilon: 0.212357, episode:   59\n",
      "frames: 54000, reward: -20.400000, loss: 0.004223, epsilon: 0.207034, episode:   60\n",
      "frames: 55000, reward: -20.400000, loss: 0.022618, epsilon: 0.201886, episode:   61\n",
      "frames: 56000, reward: -20.300000, loss: 0.006286, epsilon: 0.196906, episode:   63\n",
      "frames: 57000, reward: -20.300000, loss: 0.004309, epsilon: 0.192090, episode:   64\n",
      "frames: 58000, reward: -20.400000, loss: 0.024958, epsilon: 0.187432, episode:   65\n",
      "frames: 59000, reward: -20.400000, loss: 0.016848, epsilon: 0.182926, episode:   66\n",
      "frames: 60000, reward: -20.500000, loss: 0.030390, epsilon: 0.178569, episode:   67\n",
      "frames: 61000, reward: -20.400000, loss: 0.016256, epsilon: 0.174354, episode:   68\n",
      "frames: 62000, reward: -20.300000, loss: 0.011161, epsilon: 0.170277, episode:   69\n",
      "frames: 63000, reward: -20.200000, loss: 0.012512, epsilon: 0.166334, episode:   70\n",
      "frames: 64000, reward: -20.000000, loss: 0.022381, epsilon: 0.162520, episode:   71\n",
      "frames: 65000, reward: -19.800000, loss: 0.013650, epsilon: 0.158831, episode:   72\n",
      "frames: 66000, reward: -19.800000, loss: 0.016746, epsilon: 0.155263, episode:   73\n",
      "frames: 67000, reward: -19.500000, loss: 0.011484, epsilon: 0.151812, episode:   74\n",
      "frames: 68000, reward: -19.400000, loss: 0.006258, epsilon: 0.148474, episode:   75\n",
      "frames: 69000, reward: -19.400000, loss: 0.009568, epsilon: 0.145246, episode:   76\n",
      "frames: 70000, reward: -19.300000, loss: 0.013331, epsilon: 0.142123, episode:   77\n",
      "frames: 71000, reward: -19.600000, loss: 0.009208, epsilon: 0.139103, episode:   78\n",
      "frames: 72000, reward: -19.700000, loss: 0.007996, epsilon: 0.136182, episode:   79\n",
      "frames: 73000, reward: -19.700000, loss: 0.015526, epsilon: 0.133357, episode:   80\n",
      "frames: 74000, reward: -19.700000, loss: 0.011545, epsilon: 0.130624, episode:   80\n",
      "frames: 75000, reward: -19.600000, loss: 0.018385, epsilon: 0.127981, episode:   81\n",
      "frames: 76000, reward: -19.600000, loss: 0.011489, epsilon: 0.125424, episode:   82\n",
      "frames: 77000, reward: -19.500000, loss: 0.014783, epsilon: 0.122952, episode:   83\n",
      "frames: 78000, reward: -19.700000, loss: 0.023895, epsilon: 0.120560, episode:   84\n",
      "frames: 79000, reward: -19.500000, loss: 0.013346, epsilon: 0.118247, episode:   85\n",
      "frames: 80000, reward: -19.400000, loss: 0.018815, epsilon: 0.116009, episode:   86\n",
      "frames: 81000, reward: -19.400000, loss: 0.020356, epsilon: 0.113845, episode:   86\n",
      "frames: 82000, reward: -19.200000, loss: 0.021197, epsilon: 0.111752, episode:   87\n",
      "frames: 83000, reward: -18.800000, loss: 0.005736, epsilon: 0.109728, episode:   88\n",
      "frames: 84000, reward: -18.800000, loss: 0.016046, epsilon: 0.107770, episode:   88\n",
      "frames: 85000, reward: -18.400000, loss: 0.023605, epsilon: 0.105876, episode:   89\n",
      "frames: 86000, reward: -18.300000, loss: 0.015476, epsilon: 0.104044, episode:   90\n",
      "frames: 87000, reward: -18.300000, loss: 0.014607, epsilon: 0.102272, episode:   90\n",
      "frames: 88000, reward: -18.000000, loss: 0.009088, epsilon: 0.100558, episode:   91\n",
      "frames: 89000, reward: -18.000000, loss: 0.011567, epsilon: 0.098901, episode:   91\n",
      "frames: 90000, reward: -17.800000, loss: 0.012362, epsilon: 0.097298, episode:   92\n",
      "frames: 91000, reward: -17.800000, loss: 0.013246, epsilon: 0.095747, episode:   93\n",
      "frames: 92000, reward: -17.800000, loss: 0.010588, epsilon: 0.094247, episode:   93\n",
      "frames: 93000, reward: -17.700000, loss: 0.009500, epsilon: 0.092797, episode:   94\n",
      "frames: 94000, reward: -17.700000, loss: 0.012970, epsilon: 0.091394, episode:   94\n",
      "frames: 95000, reward: -17.700000, loss: 0.005806, epsilon: 0.090037, episode:   95\n",
      "frames: 96000, reward: -17.700000, loss: 0.009579, epsilon: 0.088724, episode:   95\n",
      "frames: 97000, reward: -17.000000, loss: 0.009840, epsilon: 0.087455, episode:   96\n",
      "frames: 98000, reward: -16.900000, loss: 0.012390, epsilon: 0.086227, episode:   97\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 99000, reward: -16.900000, loss: 0.017712, epsilon: 0.085039, episode:   97\n",
      "frames: 100000, reward: -16.900000, loss: 0.013828, epsilon: 0.083890, episode:   98\n",
      "frames: 101000, reward: -16.900000, loss: 0.014143, epsilon: 0.082779, episode:   98\n",
      "frames: 102000, reward: -16.600000, loss: 0.009617, epsilon: 0.081705, episode:   99\n",
      "frames: 103000, reward: -16.600000, loss: 0.032885, epsilon: 0.080665, episode:   99\n",
      "frames: 104000, reward: -16.400000, loss: 0.011300, epsilon: 0.079660, episode:  100\n",
      "frames: 105000, reward: -16.400000, loss: 0.011319, epsilon: 0.078688, episode:  100\n",
      "frames: 106000, reward: -16.500000, loss: 0.008643, epsilon: 0.077747, episode:  101\n",
      "frames: 107000, reward: -16.500000, loss: 0.012676, epsilon: 0.076837, episode:  101\n",
      "frames: 108000, reward: -16.200000, loss: 0.005347, epsilon: 0.075958, episode:  102\n",
      "frames: 109000, reward: -16.200000, loss: 0.011709, epsilon: 0.075107, episode:  102\n",
      "frames: 110000, reward: -16.000000, loss: 0.014729, epsilon: 0.074283, episode:  103\n",
      "frames: 111000, reward: -16.000000, loss: 0.010694, epsilon: 0.073487, episode:  103\n",
      "frames: 112000, reward: -16.000000, loss: 0.008370, epsilon: 0.072717, episode:  103\n",
      "frames: 113000, reward: -15.500000, loss: 0.021773, epsilon: 0.071973, episode:  104\n",
      "frames: 114000, reward: -15.300000, loss: 0.005428, epsilon: 0.071252, episode:  105\n",
      "frames: 115000, reward: -15.300000, loss: 0.010866, epsilon: 0.070556, episode:  105\n",
      "frames: 116000, reward: -15.800000, loss: 0.008444, epsilon: 0.069882, episode:  106\n",
      "frames: 117000, reward: -15.800000, loss: 0.006709, epsilon: 0.069230, episode:  106\n",
      "frames: 118000, reward: -15.800000, loss: 0.025859, epsilon: 0.068599, episode:  106\n",
      "frames: 119000, reward: -15.000000, loss: 0.010632, epsilon: 0.067990, episode:  107\n",
      "frames: 120000, reward: -15.000000, loss: 0.006102, epsilon: 0.067400, episode:  107\n",
      "frames: 121000, reward: -14.800000, loss: 0.029025, epsilon: 0.066829, episode:  108\n",
      "frames: 122000, reward: -14.800000, loss: 0.008782, epsilon: 0.066278, episode:  108\n",
      "frames: 123000, reward: -15.300000, loss: 0.034248, epsilon: 0.065744, episode:  109\n",
      "frames: 124000, reward: -15.300000, loss: 0.012428, epsilon: 0.065228, episode:  109\n",
      "frames: 125000, reward: -14.800000, loss: 0.008929, epsilon: 0.064729, episode:  110\n",
      "frames: 126000, reward: -14.800000, loss: 0.010244, epsilon: 0.064246, episode:  110\n",
      "frames: 127000, reward: -14.800000, loss: 0.016091, epsilon: 0.063779, episode:  110\n",
      "frames: 128000, reward: -14.300000, loss: 0.007951, epsilon: 0.063327, episode:  111\n",
      "frames: 129000, reward: -14.600000, loss: 0.028235, epsilon: 0.062890, episode:  112\n",
      "frames: 130000, reward: -14.600000, loss: 0.019799, epsilon: 0.062468, episode:  112\n",
      "frames: 131000, reward: -14.700000, loss: 0.018175, epsilon: 0.062059, episode:  113\n",
      "frames: 132000, reward: -14.700000, loss: 0.009047, epsilon: 0.061663, episode:  113\n",
      "frames: 133000, reward: -15.500000, loss: 0.009128, epsilon: 0.061281, episode:  114\n",
      "frames: 134000, reward: -15.500000, loss: 0.007828, epsilon: 0.060911, episode:  114\n",
      "frames: 135000, reward: -14.800000, loss: 0.008672, epsilon: 0.060554, episode:  115\n",
      "frames: 136000, reward: -14.800000, loss: 0.007304, epsilon: 0.060208, episode:  115\n",
      "frames: 137000, reward: -14.300000, loss: 0.013766, epsilon: 0.059873, episode:  116\n",
      "frames: 138000, reward: -14.300000, loss: 0.008932, epsilon: 0.059549, episode:  116\n",
      "frames: 139000, reward: -15.200000, loss: 0.008282, epsilon: 0.059236, episode:  117\n",
      "frames: 140000, reward: -15.200000, loss: 0.005664, epsilon: 0.058933, episode:  117\n",
      "frames: 141000, reward: -15.300000, loss: 0.017700, epsilon: 0.058641, episode:  118\n",
      "frames: 142000, reward: -15.300000, loss: 0.011092, epsilon: 0.058357, episode:  118\n",
      "frames: 143000, reward: -14.600000, loss: 0.005544, epsilon: 0.058083, episode:  119\n",
      "frames: 144000, reward: -14.600000, loss: 0.004457, epsilon: 0.057818, episode:  119\n",
      "frames: 145000, reward: -15.400000, loss: 0.013759, epsilon: 0.057562, episode:  120\n",
      "frames: 146000, reward: -15.400000, loss: 0.007208, epsilon: 0.057314, episode:  120\n",
      "frames: 147000, reward: -15.400000, loss: 0.005370, epsilon: 0.057074, episode:  120\n",
      "frames: 148000, reward: -15.200000, loss: 0.004383, epsilon: 0.056842, episode:  121\n",
      "frames: 149000, reward: -15.200000, loss: 0.006600, epsilon: 0.056618, episode:  121\n",
      "frames: 150000, reward: -15.100000, loss: 0.009842, epsilon: 0.056401, episode:  122\n",
      "frames: 151000, reward: -15.100000, loss: 0.012655, epsilon: 0.056191, episode:  122\n",
      "frames: 152000, reward: -15.000000, loss: 0.007500, epsilon: 0.055988, episode:  123\n",
      "frames: 153000, reward: -15.000000, loss: 0.011797, epsilon: 0.055792, episode:  123\n",
      "frames: 154000, reward: -14.700000, loss: 0.014604, epsilon: 0.055602, episode:  124\n",
      "frames: 155000, reward: -14.700000, loss: 0.017789, epsilon: 0.055418, episode:  124\n",
      "frames: 156000, reward: -15.600000, loss: 0.011515, epsilon: 0.055241, episode:  125\n",
      "frames: 157000, reward: -15.600000, loss: 0.014011, epsilon: 0.055069, episode:  125\n",
      "frames: 158000, reward: -15.300000, loss: 0.016234, epsilon: 0.054903, episode:  126\n",
      "frames: 159000, reward: -15.300000, loss: 0.006898, epsilon: 0.054742, episode:  126\n",
      "frames: 160000, reward: -15.300000, loss: 0.009610, epsilon: 0.054587, episode:  126\n",
      "frames: 161000, reward: -14.500000, loss: 0.017030, epsilon: 0.054436, episode:  127\n",
      "frames: 162000, reward: -14.500000, loss: 0.008976, epsilon: 0.054291, episode:  127\n",
      "frames: 163000, reward: -13.700000, loss: 0.017886, epsilon: 0.054150, episode:  128\n",
      "frames: 164000, reward: -13.700000, loss: 0.012442, epsilon: 0.054014, episode:  128\n",
      "frames: 165000, reward: -13.700000, loss: 0.009698, epsilon: 0.053882, episode:  128\n",
      "frames: 166000, reward: -13.100000, loss: 0.006219, epsilon: 0.053755, episode:  129\n",
      "frames: 167000, reward: -13.100000, loss: 0.010039, epsilon: 0.053632, episode:  129\n",
      "frames: 168000, reward: -12.400000, loss: 0.017227, epsilon: 0.053513, episode:  130\n",
      "frames: 169000, reward: -12.400000, loss: 0.005617, epsilon: 0.053398, episode:  130\n",
      "frames: 170000, reward: -12.400000, loss: 0.008562, epsilon: 0.053286, episode:  130\n",
      "frames: 171000, reward: -12.800000, loss: 0.009332, epsilon: 0.053179, episode:  131\n",
      "frames: 172000, reward: -12.800000, loss: 0.004568, epsilon: 0.053074, episode:  131\n",
      "frames: 173000, reward: -12.800000, loss: 0.011056, epsilon: 0.052974, episode:  131\n",
      "frames: 174000, reward: -12.000000, loss: 0.009854, epsilon: 0.052876, episode:  132\n",
      "frames: 175000, reward: -12.000000, loss: 0.009969, epsilon: 0.052782, episode:  132\n",
      "frames: 176000, reward: -11.500000, loss: 0.008685, epsilon: 0.052691, episode:  133\n",
      "frames: 177000, reward: -11.500000, loss: 0.012644, epsilon: 0.052602, episode:  133\n",
      "frames: 178000, reward: -11.100000, loss: 0.008505, epsilon: 0.052517, episode:  134\n",
      "frames: 179000, reward: -11.100000, loss: 0.009441, epsilon: 0.052435, episode:  134\n",
      "frames: 180000, reward: -11.100000, loss: 0.004024, epsilon: 0.052355, episode:  134\n",
      "frames: 181000, reward: -10.100000, loss: 0.009133, epsilon: 0.052278, episode:  135\n",
      "frames: 182000, reward: -10.100000, loss: 0.003980, epsilon: 0.052203, episode:  135\n",
      "frames: 183000, reward: -10.100000, loss: 0.005037, epsilon: 0.052131, episode:  135\n",
      "frames: 184000, reward: -9.200000, loss: 0.008508, epsilon: 0.052061, episode:  136\n",
      "frames: 185000, reward: -9.200000, loss: 0.020394, epsilon: 0.051993, episode:  136\n",
      "frames: 186000, reward: -9.200000, loss: 0.006981, epsilon: 0.051928, episode:  136\n",
      "frames: 187000, reward: -9.100000, loss: 0.012852, epsilon: 0.051865, episode:  137\n",
      "frames: 188000, reward: -9.100000, loss: 0.015096, epsilon: 0.051804, episode:  137\n",
      "frames: 189000, reward: -9.100000, loss: 0.009738, epsilon: 0.051744, episode:  137\n",
      "frames: 190000, reward: -8.100000, loss: 0.017128, epsilon: 0.051687, episode:  138\n",
      "frames: 191000, reward: -8.100000, loss: 0.005241, epsilon: 0.051632, episode:  138\n",
      "frames: 192000, reward: -8.100000, loss: 0.009731, epsilon: 0.051578, episode:  138\n",
      "frames: 193000, reward: -7.300000, loss: 0.005137, epsilon: 0.051527, episode:  139\n",
      "frames: 194000, reward: -7.300000, loss: 0.004546, epsilon: 0.051477, episode:  139\n",
      "frames: 195000, reward: -6.700000, loss: 0.016974, epsilon: 0.051428, episode:  140\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 196000, reward: -6.700000, loss: 0.016525, epsilon: 0.051381, episode:  140\n",
      "frames: 197000, reward: -6.700000, loss: 0.005381, epsilon: 0.051336, episode:  140\n",
      "frames: 198000, reward: -4.300000, loss: 0.004682, epsilon: 0.051292, episode:  141\n",
      "frames: 199000, reward: -4.300000, loss: 0.003945, epsilon: 0.051250, episode:  141\n",
      "frames: 200000, reward: -1.900000, loss: 0.035970, epsilon: 0.051209, episode:  142\n",
      "frames: 201000, reward: -1.900000, loss: 0.005281, epsilon: 0.051169, episode:  142\n",
      "frames: 202000, reward: 0.100000, loss: 0.012304, epsilon: 0.051131, episode:  143\n",
      "frames: 203000, reward: 0.100000, loss: 0.006703, epsilon: 0.051094, episode:  143\n",
      "frames: 204000, reward: 0.100000, loss: 0.014608, epsilon: 0.051058, episode:  143\n",
      "frames: 205000, reward: 2.500000, loss: 0.002679, epsilon: 0.051023, episode:  144\n",
      "frames: 206000, reward: 2.500000, loss: 0.004788, epsilon: 0.050990, episode:  144\n",
      "frames: 207000, reward: 4.900000, loss: 0.006269, epsilon: 0.050957, episode:  145\n",
      "frames: 208000, reward: 4.900000, loss: 0.006528, epsilon: 0.050926, episode:  145\n",
      "frames: 209000, reward: 3.700000, loss: 0.008618, epsilon: 0.050896, episode:  146\n",
      "frames: 210000, reward: 3.700000, loss: 0.004000, epsilon: 0.050866, episode:  146\n",
      "frames: 211000, reward: 3.700000, loss: 0.003358, epsilon: 0.050838, episode:  146\n",
      "frames: 212000, reward: 4.000000, loss: 0.003859, epsilon: 0.050810, episode:  147\n",
      "frames: 213000, reward: 4.000000, loss: 0.018952, epsilon: 0.050784, episode:  147\n",
      "frames: 214000, reward: 4.900000, loss: 0.011039, epsilon: 0.050758, episode:  148\n",
      "frames: 215000, reward: 4.900000, loss: 0.005138, epsilon: 0.050733, episode:  148\n",
      "frames: 216000, reward: 6.300000, loss: 0.030673, epsilon: 0.050709, episode:  149\n",
      "frames: 217000, reward: 6.300000, loss: 0.014754, epsilon: 0.050686, episode:  149\n",
      "frames: 218000, reward: 8.600000, loss: 0.004546, epsilon: 0.050664, episode:  150\n",
      "frames: 219000, reward: 8.600000, loss: 0.003495, epsilon: 0.050642, episode:  150\n",
      "frames: 220000, reward: 8.900000, loss: 0.004202, epsilon: 0.050621, episode:  151\n",
      "frames: 221000, reward: 8.900000, loss: 0.005467, epsilon: 0.050600, episode:  151\n",
      "frames: 222000, reward: 8.600000, loss: 0.003599, epsilon: 0.050581, episode:  152\n",
      "frames: 223000, reward: 8.600000, loss: 0.003792, epsilon: 0.050562, episode:  152\n",
      "frames: 224000, reward: 8.600000, loss: 0.006582, epsilon: 0.050543, episode:  152\n",
      "frames: 225000, reward: 8.800000, loss: 0.005288, epsilon: 0.050525, episode:  153\n",
      "frames: 226000, reward: 8.800000, loss: 0.012201, epsilon: 0.050508, episode:  153\n",
      "frames: 227000, reward: 8.500000, loss: 0.012093, epsilon: 0.050492, episode:  154\n",
      "frames: 228000, reward: 8.500000, loss: 0.004002, epsilon: 0.050475, episode:  154\n",
      "frames: 229000, reward: 8.700000, loss: 0.003232, epsilon: 0.050460, episode:  155\n",
      "frames: 230000, reward: 8.700000, loss: 0.028768, epsilon: 0.050445, episode:  155\n",
      "frames: 231000, reward: 10.900000, loss: 0.004773, epsilon: 0.050430, episode:  156\n",
      "frames: 232000, reward: 10.900000, loss: 0.002520, epsilon: 0.050416, episode:  156\n",
      "frames: 233000, reward: 10.900000, loss: 0.003225, epsilon: 0.050402, episode:  156\n",
      "frames: 234000, reward: 12.800000, loss: 0.002652, epsilon: 0.050389, episode:  157\n",
      "frames: 235000, reward: 12.800000, loss: 0.005182, epsilon: 0.050376, episode:  157\n",
      "frames: 236000, reward: 12.500000, loss: 0.003160, epsilon: 0.050364, episode:  158\n",
      "frames: 237000, reward: 12.500000, loss: 0.003695, epsilon: 0.050352, episode:  158\n",
      "frames: 238000, reward: 12.400000, loss: 0.004367, epsilon: 0.050341, episode:  159\n",
      "frames: 239000, reward: 12.400000, loss: 0.002485, epsilon: 0.050329, episode:  159\n",
      "frames: 240000, reward: 12.400000, loss: 0.010288, epsilon: 0.050319, episode:  159\n",
      "frames: 241000, reward: 11.600000, loss: 0.005405, epsilon: 0.050308, episode:  160\n",
      "frames: 242000, reward: 11.600000, loss: 0.004351, epsilon: 0.050298, episode:  160\n",
      "frames: 243000, reward: 11.600000, loss: 0.004691, epsilon: 0.050288, episode:  160\n",
      "frames: 244000, reward: 10.600000, loss: 0.003569, epsilon: 0.050279, episode:  161\n",
      "frames: 245000, reward: 10.600000, loss: 0.004136, epsilon: 0.050270, episode:  161\n",
      "frames: 246000, reward: 10.800000, loss: 0.004027, epsilon: 0.050261, episode:  162\n",
      "frames: 247000, reward: 10.800000, loss: 0.001712, epsilon: 0.050252, episode:  162\n",
      "frames: 248000, reward: 10.900000, loss: 0.002989, epsilon: 0.050244, episode:  163\n",
      "frames: 249000, reward: 10.900000, loss: 0.003940, epsilon: 0.050236, episode:  163\n",
      "frames: 250000, reward: 11.500000, loss: 0.006980, epsilon: 0.050228, episode:  164\n",
      "frames: 251000, reward: 11.500000, loss: 0.003620, epsilon: 0.050221, episode:  164\n",
      "frames: 252000, reward: 11.200000, loss: 0.007754, epsilon: 0.050214, episode:  165\n",
      "frames: 253000, reward: 11.200000, loss: 0.017083, epsilon: 0.050207, episode:  165\n",
      "frames: 254000, reward: 11.800000, loss: 0.006720, epsilon: 0.050200, episode:  166\n",
      "frames: 255000, reward: 11.800000, loss: 0.002854, epsilon: 0.050193, episode:  166\n",
      "frames: 256000, reward: 12.000000, loss: 0.007801, epsilon: 0.050187, episode:  167\n",
      "frames: 257000, reward: 12.000000, loss: 0.002229, epsilon: 0.050181, episode:  167\n",
      "frames: 258000, reward: 12.000000, loss: 0.003183, epsilon: 0.050175, episode:  167\n",
      "frames: 259000, reward: 12.700000, loss: 0.010615, epsilon: 0.050169, episode:  168\n",
      "frames: 260000, reward: 12.700000, loss: 0.005541, epsilon: 0.050164, episode:  168\n",
      "frames: 261000, reward: 12.800000, loss: 0.003470, epsilon: 0.050158, episode:  169\n",
      "frames: 262000, reward: 12.800000, loss: 0.002395, epsilon: 0.050153, episode:  169\n",
      "frames: 263000, reward: 13.500000, loss: 0.007928, epsilon: 0.050148, episode:  170\n",
      "frames: 264000, reward: 13.500000, loss: 0.003467, epsilon: 0.050143, episode:  170\n",
      "frames: 265000, reward: 14.600000, loss: 0.001874, epsilon: 0.050139, episode:  171\n",
      "frames: 266000, reward: 14.600000, loss: 0.002565, epsilon: 0.050134, episode:  171\n",
      "frames: 267000, reward: 14.500000, loss: 0.004181, epsilon: 0.050130, episode:  172\n",
      "frames: 268000, reward: 14.500000, loss: 0.007112, epsilon: 0.050125, episode:  172\n",
      "frames: 269000, reward: 14.700000, loss: 0.021062, epsilon: 0.050121, episode:  173\n",
      "frames: 270000, reward: 14.700000, loss: 0.003758, epsilon: 0.050117, episode:  173\n",
      "frames: 271000, reward: 15.000000, loss: 0.006574, epsilon: 0.050113, episode:  174\n",
      "frames: 272000, reward: 15.000000, loss: 0.015661, epsilon: 0.050110, episode:  174\n",
      "frames: 273000, reward: 15.000000, loss: 0.003547, epsilon: 0.050106, episode:  175\n",
      "frames: 274000, reward: 15.000000, loss: 0.003347, epsilon: 0.050103, episode:  175\n",
      "frames: 275000, reward: 14.700000, loss: 0.004256, epsilon: 0.050099, episode:  176\n",
      "frames: 276000, reward: 14.700000, loss: 0.004694, epsilon: 0.050096, episode:  176\n",
      "frames: 277000, reward: 14.800000, loss: 0.002425, epsilon: 0.050093, episode:  177\n",
      "frames: 278000, reward: 14.800000, loss: 0.003886, epsilon: 0.050090, episode:  177\n",
      "frames: 279000, reward: 14.800000, loss: 0.003362, epsilon: 0.050087, episode:  177\n",
      "frames: 280000, reward: 13.900000, loss: 0.003832, epsilon: 0.050084, episode:  178\n",
      "frames: 281000, reward: 13.900000, loss: 0.006431, epsilon: 0.050081, episode:  178\n",
      "frames: 282000, reward: 13.900000, loss: 0.002277, epsilon: 0.050079, episode:  179\n",
      "frames: 283000, reward: 13.900000, loss: 0.006377, epsilon: 0.050076, episode:  179\n",
      "frames: 284000, reward: 12.500000, loss: 0.004952, epsilon: 0.050074, episode:  180\n",
      "frames: 285000, reward: 12.500000, loss: 0.002425, epsilon: 0.050071, episode:  180\n",
      "frames: 286000, reward: 12.500000, loss: 0.009812, epsilon: 0.050069, episode:  180\n",
      "frames: 287000, reward: 12.100000, loss: 0.008066, epsilon: 0.050067, episode:  181\n",
      "frames: 288000, reward: 12.100000, loss: 0.006810, epsilon: 0.050064, episode:  181\n",
      "frames: 289000, reward: 12.500000, loss: 0.002831, epsilon: 0.050062, episode:  182\n",
      "frames: 290000, reward: 12.500000, loss: 0.004594, epsilon: 0.050060, episode:  182\n",
      "frames: 291000, reward: 12.800000, loss: 0.006203, epsilon: 0.050058, episode:  183\n",
      "frames: 292000, reward: 12.800000, loss: 0.005400, epsilon: 0.050056, episode:  183\n",
      "frames: 293000, reward: 11.800000, loss: 0.002759, epsilon: 0.050054, episode:  184\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 294000, reward: 11.800000, loss: 0.003396, epsilon: 0.050053, episode:  184\n",
      "frames: 295000, reward: 11.700000, loss: 0.003191, epsilon: 0.050051, episode:  185\n",
      "frames: 296000, reward: 11.700000, loss: 0.005840, epsilon: 0.050049, episode:  185\n",
      "frames: 297000, reward: 12.300000, loss: 0.015511, epsilon: 0.050048, episode:  186\n",
      "frames: 298000, reward: 12.300000, loss: 0.002289, epsilon: 0.050046, episode:  186\n",
      "frames: 299000, reward: 12.100000, loss: 0.001891, epsilon: 0.050045, episode:  187\n",
      "frames: 300000, reward: 12.100000, loss: 0.004743, epsilon: 0.050043, episode:  187\n",
      "frames: 301000, reward: 13.400000, loss: 0.004178, epsilon: 0.050042, episode:  188\n",
      "frames: 302000, reward: 13.400000, loss: 0.003493, epsilon: 0.050040, episode:  188\n",
      "frames: 303000, reward: 13.100000, loss: 0.008947, epsilon: 0.050039, episode:  189\n",
      "frames: 304000, reward: 13.100000, loss: 0.002428, epsilon: 0.050038, episode:  189\n",
      "frames: 305000, reward: 14.300000, loss: 0.010058, epsilon: 0.050037, episode:  190\n",
      "frames: 306000, reward: 14.300000, loss: 0.003811, epsilon: 0.050035, episode:  190\n",
      "frames: 307000, reward: 14.700000, loss: 0.003021, epsilon: 0.050034, episode:  191\n",
      "frames: 308000, reward: 14.700000, loss: 0.002557, epsilon: 0.050033, episode:  191\n",
      "frames: 309000, reward: 14.800000, loss: 0.005299, epsilon: 0.050032, episode:  192\n",
      "frames: 310000, reward: 14.800000, loss: 0.003298, epsilon: 0.050031, episode:  192\n",
      "frames: 311000, reward: 14.700000, loss: 0.002748, epsilon: 0.050030, episode:  193\n",
      "frames: 312000, reward: 14.700000, loss: 0.002784, epsilon: 0.050029, episode:  193\n",
      "frames: 313000, reward: 16.000000, loss: 0.002780, epsilon: 0.050028, episode:  194\n",
      "frames: 314000, reward: 16.000000, loss: 0.004081, epsilon: 0.050027, episode:  194\n",
      "frames: 315000, reward: 16.200000, loss: 0.002716, epsilon: 0.050026, episode:  195\n",
      "frames: 316000, reward: 16.200000, loss: 0.003423, epsilon: 0.050025, episode:  195\n",
      "frames: 317000, reward: 15.500000, loss: 0.004012, epsilon: 0.050024, episode:  196\n",
      "frames: 318000, reward: 15.500000, loss: 0.002912, epsilon: 0.050024, episode:  196\n",
      "frames: 319000, reward: 15.500000, loss: 0.003309, epsilon: 0.050023, episode:  196\n",
      "frames: 320000, reward: 15.400000, loss: 0.003764, epsilon: 0.050022, episode:  197\n",
      "frames: 321000, reward: 15.400000, loss: 0.002818, epsilon: 0.050021, episode:  197\n",
      "frames: 322000, reward: 14.900000, loss: 0.002460, epsilon: 0.050021, episode:  198\n",
      "frames: 323000, reward: 14.900000, loss: 0.003057, epsilon: 0.050020, episode:  198\n",
      "frames: 324000, reward: 15.200000, loss: 0.011180, epsilon: 0.050019, episode:  199\n",
      "frames: 325000, reward: 15.200000, loss: 0.001802, epsilon: 0.050019, episode:  199\n",
      "frames: 326000, reward: 15.600000, loss: 0.005656, epsilon: 0.050018, episode:  200\n",
      "frames: 327000, reward: 15.600000, loss: 0.003441, epsilon: 0.050018, episode:  200\n",
      "frames: 328000, reward: 15.900000, loss: 0.003991, epsilon: 0.050017, episode:  201\n",
      "frames: 329000, reward: 15.900000, loss: 0.005343, epsilon: 0.050016, episode:  201\n",
      "frames: 330000, reward: 15.500000, loss: 0.002842, epsilon: 0.050016, episode:  202\n",
      "frames: 331000, reward: 15.500000, loss: 0.002744, epsilon: 0.050015, episode:  202\n",
      "frames: 332000, reward: 15.400000, loss: 0.002574, epsilon: 0.050015, episode:  203\n",
      "frames: 333000, reward: 15.400000, loss: 0.003571, epsilon: 0.050014, episode:  203\n",
      "frames: 334000, reward: 15.100000, loss: 0.004958, epsilon: 0.050014, episode:  204\n",
      "frames: 335000, reward: 15.100000, loss: 0.004808, epsilon: 0.050013, episode:  204\n",
      "frames: 336000, reward: 15.200000, loss: 0.014048, epsilon: 0.050013, episode:  205\n",
      "frames: 337000, reward: 15.200000, loss: 0.002030, epsilon: 0.050013, episode:  205\n",
      "frames: 338000, reward: 15.700000, loss: 0.003710, epsilon: 0.050012, episode:  206\n",
      "frames: 339000, reward: 15.700000, loss: 0.001449, epsilon: 0.050012, episode:  206\n",
      "frames: 340000, reward: 16.000000, loss: 0.002036, epsilon: 0.050011, episode:  207\n",
      "frames: 341000, reward: 16.000000, loss: 0.001640, epsilon: 0.050011, episode:  207\n",
      "frames: 342000, reward: 16.200000, loss: 0.011635, epsilon: 0.050011, episode:  208\n",
      "frames: 343000, reward: 16.200000, loss: 0.009611, epsilon: 0.050010, episode:  208\n",
      "frames: 344000, reward: 15.200000, loss: 0.003113, epsilon: 0.050010, episode:  209\n",
      "frames: 345000, reward: 15.200000, loss: 0.004349, epsilon: 0.050010, episode:  209\n",
      "frames: 346000, reward: 15.000000, loss: 0.002220, epsilon: 0.050009, episode:  210\n",
      "frames: 347000, reward: 15.000000, loss: 0.005960, epsilon: 0.050009, episode:  210\n",
      "frames: 348000, reward: 14.800000, loss: 0.004451, epsilon: 0.050009, episode:  211\n",
      "frames: 349000, reward: 14.800000, loss: 0.002303, epsilon: 0.050008, episode:  211\n",
      "frames: 350000, reward: 14.600000, loss: 0.008323, epsilon: 0.050008, episode:  212\n",
      "frames: 351000, reward: 14.600000, loss: 0.002250, epsilon: 0.050008, episode:  212\n",
      "frames: 352000, reward: 14.800000, loss: 0.000920, epsilon: 0.050008, episode:  213\n",
      "frames: 353000, reward: 14.800000, loss: 0.001464, epsilon: 0.050007, episode:  213\n",
      "frames: 354000, reward: 15.000000, loss: 0.005137, epsilon: 0.050007, episode:  214\n",
      "frames: 355000, reward: 15.000000, loss: 0.002860, epsilon: 0.050007, episode:  214\n",
      "frames: 356000, reward: 15.300000, loss: 0.001888, epsilon: 0.050007, episode:  215\n",
      "frames: 357000, reward: 15.300000, loss: 0.005199, epsilon: 0.050006, episode:  215\n",
      "frames: 358000, reward: 15.300000, loss: 0.002076, epsilon: 0.050006, episode:  216\n",
      "frames: 359000, reward: 15.300000, loss: 0.004839, epsilon: 0.050006, episode:  216\n",
      "frames: 360000, reward: 15.000000, loss: 0.001909, epsilon: 0.050006, episode:  217\n",
      "frames: 361000, reward: 15.000000, loss: 0.005396, epsilon: 0.050006, episode:  217\n",
      "frames: 362000, reward: 14.800000, loss: 0.002392, epsilon: 0.050005, episode:  218\n",
      "frames: 363000, reward: 14.800000, loss: 0.003712, epsilon: 0.050005, episode:  218\n",
      "frames: 364000, reward: 14.800000, loss: 0.001196, epsilon: 0.050005, episode:  218\n",
      "frames: 365000, reward: 15.700000, loss: 0.003936, epsilon: 0.050005, episode:  219\n",
      "frames: 366000, reward: 15.700000, loss: 0.002696, epsilon: 0.050005, episode:  219\n",
      "frames: 367000, reward: 15.400000, loss: 0.003537, epsilon: 0.050005, episode:  220\n",
      "frames: 368000, reward: 15.400000, loss: 0.002058, epsilon: 0.050004, episode:  220\n",
      "frames: 369000, reward: 15.400000, loss: 0.003113, epsilon: 0.050004, episode:  221\n",
      "frames: 370000, reward: 15.400000, loss: 0.004552, epsilon: 0.050004, episode:  221\n",
      "frames: 371000, reward: 15.400000, loss: 0.001099, epsilon: 0.050004, episode:  222\n",
      "frames: 372000, reward: 15.400000, loss: 0.006227, epsilon: 0.050004, episode:  222\n",
      "frames: 373000, reward: 15.400000, loss: 0.007333, epsilon: 0.050004, episode:  222\n",
      "frames: 374000, reward: 14.200000, loss: 0.004235, epsilon: 0.050004, episode:  223\n",
      "frames: 375000, reward: 14.200000, loss: 0.004295, epsilon: 0.050004, episode:  223\n",
      "frames: 376000, reward: 13.600000, loss: 0.004928, epsilon: 0.050003, episode:  224\n",
      "frames: 377000, reward: 13.600000, loss: 0.008666, epsilon: 0.050003, episode:  224\n",
      "frames: 378000, reward: 13.300000, loss: 0.003650, epsilon: 0.050003, episode:  225\n",
      "frames: 379000, reward: 13.300000, loss: 0.005919, epsilon: 0.050003, episode:  225\n",
      "frames: 380000, reward: 13.400000, loss: 0.003317, epsilon: 0.050003, episode:  226\n",
      "frames: 381000, reward: 13.400000, loss: 0.001634, epsilon: 0.050003, episode:  226\n",
      "frames: 382000, reward: 13.600000, loss: 0.003149, epsilon: 0.050003, episode:  227\n",
      "frames: 383000, reward: 13.600000, loss: 0.002911, epsilon: 0.050003, episode:  227\n",
      "frames: 384000, reward: 13.300000, loss: 0.002321, epsilon: 0.050003, episode:  228\n",
      "frames: 385000, reward: 13.300000, loss: 0.004391, epsilon: 0.050003, episode:  228\n",
      "frames: 386000, reward: 13.500000, loss: 0.002998, epsilon: 0.050002, episode:  229\n",
      "frames: 387000, reward: 13.500000, loss: 0.004992, epsilon: 0.050002, episode:  229\n",
      "frames: 388000, reward: 13.500000, loss: 0.004979, epsilon: 0.050002, episode:  229\n",
      "frames: 389000, reward: 13.300000, loss: 0.002434, epsilon: 0.050002, episode:  230\n",
      "frames: 390000, reward: 13.300000, loss: 0.003495, epsilon: 0.050002, episode:  230\n",
      "frames: 391000, reward: 12.800000, loss: 0.006498, epsilon: 0.050002, episode:  231\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 392000, reward: 12.800000, loss: 0.004464, epsilon: 0.050002, episode:  231\n",
      "frames: 393000, reward: 13.100000, loss: 0.003384, epsilon: 0.050002, episode:  232\n",
      "frames: 394000, reward: 13.100000, loss: 0.003640, epsilon: 0.050002, episode:  232\n",
      "frames: 395000, reward: 14.300000, loss: 0.003016, epsilon: 0.050002, episode:  233\n",
      "frames: 396000, reward: 14.300000, loss: 0.007654, epsilon: 0.050002, episode:  233\n",
      "frames: 397000, reward: 14.600000, loss: 0.006500, epsilon: 0.050002, episode:  234\n",
      "frames: 398000, reward: 14.600000, loss: 0.001966, epsilon: 0.050002, episode:  234\n",
      "frames: 399000, reward: 14.600000, loss: 0.002414, epsilon: 0.050002, episode:  235\n",
      "frames: 400000, reward: 14.600000, loss: 0.004889, epsilon: 0.050002, episode:  235\n",
      "frames: 401000, reward: 14.600000, loss: 0.002120, epsilon: 0.050001, episode:  236\n",
      "frames: 402000, reward: 14.600000, loss: 0.004138, epsilon: 0.050001, episode:  236\n",
      "frames: 403000, reward: 14.800000, loss: 0.001605, epsilon: 0.050001, episode:  237\n",
      "frames: 404000, reward: 14.800000, loss: 0.003261, epsilon: 0.050001, episode:  237\n",
      "frames: 405000, reward: 15.000000, loss: 0.003087, epsilon: 0.050001, episode:  238\n",
      "frames: 406000, reward: 15.000000, loss: 0.001975, epsilon: 0.050001, episode:  238\n",
      "frames: 407000, reward: 15.000000, loss: 0.003105, epsilon: 0.050001, episode:  238\n",
      "frames: 408000, reward: 14.600000, loss: 0.004836, epsilon: 0.050001, episode:  239\n",
      "frames: 409000, reward: 14.600000, loss: 0.001767, epsilon: 0.050001, episode:  239\n",
      "frames: 410000, reward: 15.100000, loss: 0.004473, epsilon: 0.050001, episode:  240\n",
      "frames: 411000, reward: 15.100000, loss: 0.001630, epsilon: 0.050001, episode:  240\n",
      "frames: 412000, reward: 15.400000, loss: 0.002141, epsilon: 0.050001, episode:  241\n",
      "frames: 413000, reward: 15.400000, loss: 0.001609, epsilon: 0.050001, episode:  241\n",
      "frames: 414000, reward: 14.500000, loss: 0.001514, epsilon: 0.050001, episode:  242\n",
      "frames: 415000, reward: 14.500000, loss: 0.004256, epsilon: 0.050001, episode:  242\n",
      "frames: 416000, reward: 14.600000, loss: 0.002568, epsilon: 0.050001, episode:  243\n",
      "frames: 417000, reward: 14.600000, loss: 0.003709, epsilon: 0.050001, episode:  243\n",
      "frames: 418000, reward: 14.400000, loss: 0.004586, epsilon: 0.050001, episode:  244\n",
      "frames: 419000, reward: 14.400000, loss: 0.005077, epsilon: 0.050001, episode:  244\n",
      "frames: 420000, reward: 14.400000, loss: 0.003321, epsilon: 0.050001, episode:  244\n",
      "frames: 421000, reward: 14.200000, loss: 0.002478, epsilon: 0.050001, episode:  245\n",
      "frames: 422000, reward: 14.200000, loss: 0.002487, epsilon: 0.050001, episode:  245\n",
      "frames: 423000, reward: 13.800000, loss: 0.004204, epsilon: 0.050001, episode:  246\n",
      "frames: 424000, reward: 13.800000, loss: 0.005094, epsilon: 0.050001, episode:  246\n",
      "frames: 425000, reward: 13.600000, loss: 0.010891, epsilon: 0.050001, episode:  247\n",
      "frames: 426000, reward: 13.600000, loss: 0.002439, epsilon: 0.050001, episode:  247\n",
      "frames: 427000, reward: 13.700000, loss: 0.007206, epsilon: 0.050001, episode:  248\n",
      "frames: 428000, reward: 13.700000, loss: 0.007028, epsilon: 0.050001, episode:  248\n",
      "frames: 429000, reward: 13.400000, loss: 0.003258, epsilon: 0.050001, episode:  249\n",
      "frames: 430000, reward: 13.400000, loss: 0.004941, epsilon: 0.050001, episode:  249\n",
      "frames: 431000, reward: 13.400000, loss: 0.001575, epsilon: 0.050001, episode:  250\n",
      "frames: 432000, reward: 13.400000, loss: 0.002484, epsilon: 0.050001, episode:  250\n",
      "frames: 433000, reward: 13.500000, loss: 0.004294, epsilon: 0.050001, episode:  251\n",
      "frames: 434000, reward: 13.500000, loss: 0.008620, epsilon: 0.050000, episode:  251\n",
      "frames: 435000, reward: 14.500000, loss: 0.004971, epsilon: 0.050000, episode:  252\n",
      "frames: 436000, reward: 14.500000, loss: 0.004688, epsilon: 0.050000, episode:  252\n",
      "frames: 437000, reward: 14.100000, loss: 0.005733, epsilon: 0.050000, episode:  253\n",
      "frames: 438000, reward: 14.100000, loss: 0.020164, epsilon: 0.050000, episode:  253\n",
      "frames: 439000, reward: 14.600000, loss: 0.005490, epsilon: 0.050000, episode:  254\n",
      "frames: 440000, reward: 14.600000, loss: 0.015624, epsilon: 0.050000, episode:  254\n",
      "frames: 441000, reward: 14.400000, loss: 0.001800, epsilon: 0.050000, episode:  255\n",
      "frames: 442000, reward: 14.400000, loss: 0.002429, epsilon: 0.050000, episode:  255\n",
      "frames: 443000, reward: 14.400000, loss: 0.005735, epsilon: 0.050000, episode:  255\n",
      "frames: 444000, reward: 13.600000, loss: 0.000773, epsilon: 0.050000, episode:  256\n",
      "frames: 445000, reward: 13.600000, loss: 0.005827, epsilon: 0.050000, episode:  256\n",
      "frames: 446000, reward: 13.200000, loss: 0.015804, epsilon: 0.050000, episode:  257\n",
      "frames: 447000, reward: 13.200000, loss: 0.001865, epsilon: 0.050000, episode:  257\n",
      "frames: 448000, reward: 13.200000, loss: 0.003718, epsilon: 0.050000, episode:  257\n",
      "frames: 449000, reward: 13.000000, loss: 0.004742, epsilon: 0.050000, episode:  258\n",
      "frames: 450000, reward: 13.000000, loss: 0.002487, epsilon: 0.050000, episode:  258\n",
      "frames: 451000, reward: 13.400000, loss: 0.003516, epsilon: 0.050000, episode:  259\n",
      "frames: 452000, reward: 13.400000, loss: 0.001686, epsilon: 0.050000, episode:  259\n",
      "frames: 453000, reward: 12.400000, loss: 0.003813, epsilon: 0.050000, episode:  260\n",
      "frames: 454000, reward: 12.400000, loss: 0.006286, epsilon: 0.050000, episode:  260\n",
      "frames: 455000, reward: 12.600000, loss: 0.003222, epsilon: 0.050000, episode:  261\n",
      "frames: 456000, reward: 12.600000, loss: 0.004432, epsilon: 0.050000, episode:  261\n",
      "frames: 457000, reward: 12.100000, loss: 0.003955, epsilon: 0.050000, episode:  262\n",
      "frames: 458000, reward: 12.100000, loss: 0.004422, epsilon: 0.050000, episode:  262\n",
      "frames: 459000, reward: 12.600000, loss: 0.002368, epsilon: 0.050000, episode:  263\n",
      "frames: 460000, reward: 12.600000, loss: 0.004546, epsilon: 0.050000, episode:  263\n",
      "frames: 461000, reward: 12.200000, loss: 0.001871, epsilon: 0.050000, episode:  264\n",
      "frames: 462000, reward: 12.200000, loss: 0.002138, epsilon: 0.050000, episode:  264\n",
      "frames: 463000, reward: 12.600000, loss: 0.002957, epsilon: 0.050000, episode:  265\n",
      "frames: 464000, reward: 12.600000, loss: 0.001472, epsilon: 0.050000, episode:  265\n",
      "frames: 465000, reward: 12.600000, loss: 0.003587, epsilon: 0.050000, episode:  265\n",
      "frames: 466000, reward: 13.300000, loss: 0.002873, epsilon: 0.050000, episode:  266\n",
      "frames: 467000, reward: 13.300000, loss: 0.002721, epsilon: 0.050000, episode:  266\n",
      "frames: 468000, reward: 13.200000, loss: 0.005601, epsilon: 0.050000, episode:  267\n",
      "frames: 469000, reward: 13.200000, loss: 0.003369, epsilon: 0.050000, episode:  267\n",
      "frames: 470000, reward: 13.300000, loss: 0.005045, epsilon: 0.050000, episode:  268\n",
      "frames: 471000, reward: 13.300000, loss: 0.010910, epsilon: 0.050000, episode:  268\n",
      "frames: 472000, reward: 13.000000, loss: 0.004714, epsilon: 0.050000, episode:  269\n",
      "frames: 473000, reward: 13.000000, loss: 0.008267, epsilon: 0.050000, episode:  269\n",
      "frames: 474000, reward: 13.000000, loss: 0.007830, epsilon: 0.050000, episode:  269\n",
      "frames: 475000, reward: 13.500000, loss: 0.003335, epsilon: 0.050000, episode:  270\n",
      "frames: 476000, reward: 13.500000, loss: 0.009352, epsilon: 0.050000, episode:  270\n",
      "frames: 477000, reward: 12.500000, loss: 0.006064, epsilon: 0.050000, episode:  271\n",
      "frames: 478000, reward: 12.500000, loss: 0.010769, epsilon: 0.050000, episode:  271\n",
      "frames: 479000, reward: 13.000000, loss: 0.001364, epsilon: 0.050000, episode:  272\n",
      "frames: 480000, reward: 13.000000, loss: 0.003219, epsilon: 0.050000, episode:  272\n",
      "frames: 481000, reward: 13.000000, loss: 0.004576, epsilon: 0.050000, episode:  272\n",
      "frames: 482000, reward: 10.000000, loss: 0.002871, epsilon: 0.050000, episode:  273\n",
      "frames: 483000, reward: 10.000000, loss: 0.005996, epsilon: 0.050000, episode:  273\n",
      "frames: 484000, reward: 9.400000, loss: 0.006294, epsilon: 0.050000, episode:  274\n",
      "frames: 485000, reward: 9.400000, loss: 0.003046, epsilon: 0.050000, episode:  274\n",
      "frames: 486000, reward: 8.500000, loss: 0.004284, epsilon: 0.050000, episode:  275\n",
      "frames: 487000, reward: 8.500000, loss: 0.007063, epsilon: 0.050000, episode:  275\n",
      "frames: 488000, reward: 9.300000, loss: 0.002658, epsilon: 0.050000, episode:  276\n",
      "frames: 489000, reward: 9.300000, loss: 0.002237, epsilon: 0.050000, episode:  276\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 490000, reward: 10.100000, loss: 0.008134, epsilon: 0.050000, episode:  277\n",
      "frames: 491000, reward: 10.100000, loss: 0.005252, epsilon: 0.050000, episode:  277\n",
      "frames: 492000, reward: 10.100000, loss: 0.005369, epsilon: 0.050000, episode:  277\n",
      "frames: 493000, reward: 9.200000, loss: 0.001682, epsilon: 0.050000, episode:  278\n",
      "frames: 494000, reward: 9.200000, loss: 0.003096, epsilon: 0.050000, episode:  278\n",
      "frames: 495000, reward: 9.800000, loss: 0.002031, epsilon: 0.050000, episode:  279\n",
      "frames: 496000, reward: 9.800000, loss: 0.005126, epsilon: 0.050000, episode:  279\n",
      "frames: 497000, reward: 10.300000, loss: 0.005505, epsilon: 0.050000, episode:  280\n",
      "frames: 498000, reward: 10.300000, loss: 0.006418, epsilon: 0.050000, episode:  280\n",
      "frames: 499000, reward: 10.600000, loss: 0.004051, epsilon: 0.050000, episode:  281\n",
      "frames: 500000, reward: 10.600000, loss: 0.010199, epsilon: 0.050000, episode:  281\n",
      "frames: 501000, reward: 9.500000, loss: 0.005550, epsilon: 0.050000, episode:  282\n",
      "frames: 502000, reward: 9.500000, loss: 0.002719, epsilon: 0.050000, episode:  282\n",
      "frames: 503000, reward: 9.500000, loss: 0.004085, epsilon: 0.050000, episode:  282\n",
      "frames: 504000, reward: 11.900000, loss: 0.002279, epsilon: 0.050000, episode:  283\n",
      "frames: 505000, reward: 11.900000, loss: 0.006439, epsilon: 0.050000, episode:  283\n",
      "frames: 506000, reward: 12.800000, loss: 0.005045, epsilon: 0.050000, episode:  284\n",
      "frames: 507000, reward: 12.800000, loss: 0.005322, epsilon: 0.050000, episode:  284\n",
      "frames: 508000, reward: 13.000000, loss: 0.006048, epsilon: 0.050000, episode:  285\n",
      "frames: 509000, reward: 13.000000, loss: 0.003559, epsilon: 0.050000, episode:  285\n",
      "frames: 510000, reward: 13.000000, loss: 0.002478, epsilon: 0.050000, episode:  285\n",
      "frames: 511000, reward: 11.600000, loss: 0.003775, epsilon: 0.050000, episode:  286\n",
      "frames: 512000, reward: 11.600000, loss: 0.006885, epsilon: 0.050000, episode:  286\n",
      "frames: 513000, reward: 10.300000, loss: 0.002411, epsilon: 0.050000, episode:  287\n",
      "frames: 514000, reward: 10.300000, loss: 0.009912, epsilon: 0.050000, episode:  287\n",
      "frames: 515000, reward: 10.300000, loss: 0.006534, epsilon: 0.050000, episode:  287\n",
      "frames: 516000, reward: 10.600000, loss: 0.005213, epsilon: 0.050000, episode:  288\n",
      "frames: 517000, reward: 10.600000, loss: 0.003566, epsilon: 0.050000, episode:  288\n",
      "frames: 518000, reward: 10.200000, loss: 0.004486, epsilon: 0.050000, episode:  289\n",
      "frames: 519000, reward: 10.200000, loss: 0.006601, epsilon: 0.050000, episode:  289\n",
      "frames: 520000, reward: 10.300000, loss: 0.002258, epsilon: 0.050000, episode:  290\n",
      "frames: 521000, reward: 10.300000, loss: 0.007814, epsilon: 0.050000, episode:  290\n",
      "frames: 522000, reward: 10.300000, loss: 0.003367, epsilon: 0.050000, episode:  290\n",
      "frames: 523000, reward: 9.800000, loss: 0.002860, epsilon: 0.050000, episode:  291\n",
      "frames: 524000, reward: 9.800000, loss: 0.005104, epsilon: 0.050000, episode:  291\n",
      "frames: 525000, reward: 9.900000, loss: 0.003987, epsilon: 0.050000, episode:  292\n",
      "frames: 526000, reward: 9.900000, loss: 0.002837, epsilon: 0.050000, episode:  292\n",
      "frames: 527000, reward: 10.300000, loss: 0.002437, epsilon: 0.050000, episode:  293\n",
      "frames: 528000, reward: 10.300000, loss: 0.005960, epsilon: 0.050000, episode:  293\n",
      "frames: 529000, reward: 10.200000, loss: 0.003218, epsilon: 0.050000, episode:  294\n",
      "frames: 530000, reward: 10.200000, loss: 0.003345, epsilon: 0.050000, episode:  294\n",
      "frames: 531000, reward: 10.700000, loss: 0.001548, epsilon: 0.050000, episode:  295\n",
      "frames: 532000, reward: 10.700000, loss: 0.010203, epsilon: 0.050000, episode:  295\n",
      "frames: 533000, reward: 9.800000, loss: 0.003285, epsilon: 0.050000, episode:  296\n",
      "frames: 534000, reward: 9.800000, loss: 0.003459, epsilon: 0.050000, episode:  296\n",
      "frames: 535000, reward: 9.800000, loss: 0.003812, epsilon: 0.050000, episode:  296\n",
      "frames: 536000, reward: 9.400000, loss: 0.005817, epsilon: 0.050000, episode:  297\n",
      "frames: 537000, reward: 9.400000, loss: 0.019420, epsilon: 0.050000, episode:  297\n",
      "frames: 538000, reward: 10.000000, loss: 0.006048, epsilon: 0.050000, episode:  298\n",
      "frames: 539000, reward: 10.000000, loss: 0.010504, epsilon: 0.050000, episode:  298\n",
      "frames: 540000, reward: 9.900000, loss: 0.002100, epsilon: 0.050000, episode:  299\n",
      "frames: 541000, reward: 9.900000, loss: 0.007426, epsilon: 0.050000, episode:  299\n",
      "frames: 542000, reward: 9.900000, loss: 0.005672, epsilon: 0.050000, episode:  299\n",
      "frames: 543000, reward: 9.400000, loss: 0.006390, epsilon: 0.050000, episode:  300\n",
      "frames: 544000, reward: 9.400000, loss: 0.007338, epsilon: 0.050000, episode:  300\n",
      "frames: 545000, reward: 10.300000, loss: 0.008823, epsilon: 0.050000, episode:  301\n",
      "frames: 546000, reward: 10.300000, loss: 0.002357, epsilon: 0.050000, episode:  301\n",
      "frames: 547000, reward: 11.200000, loss: 0.011417, epsilon: 0.050000, episode:  302\n",
      "frames: 548000, reward: 11.200000, loss: 0.004460, epsilon: 0.050000, episode:  302\n",
      "frames: 549000, reward: 10.900000, loss: 0.001594, epsilon: 0.050000, episode:  303\n",
      "frames: 550000, reward: 10.900000, loss: 0.002847, epsilon: 0.050000, episode:  303\n",
      "frames: 551000, reward: 10.900000, loss: 0.004581, epsilon: 0.050000, episode:  304\n",
      "frames: 552000, reward: 10.900000, loss: 0.003520, epsilon: 0.050000, episode:  304\n",
      "frames: 553000, reward: 10.800000, loss: 0.003406, epsilon: 0.050000, episode:  305\n",
      "frames: 554000, reward: 10.800000, loss: 0.005362, epsilon: 0.050000, episode:  305\n",
      "frames: 555000, reward: 12.400000, loss: 0.002414, epsilon: 0.050000, episode:  306\n",
      "frames: 556000, reward: 12.400000, loss: 0.005165, epsilon: 0.050000, episode:  306\n",
      "frames: 557000, reward: 14.000000, loss: 0.004923, epsilon: 0.050000, episode:  307\n",
      "frames: 558000, reward: 14.000000, loss: 0.004588, epsilon: 0.050000, episode:  307\n",
      "frames: 559000, reward: 14.400000, loss: 0.003566, epsilon: 0.050000, episode:  308\n",
      "frames: 560000, reward: 14.400000, loss: 0.002208, epsilon: 0.050000, episode:  308\n",
      "frames: 561000, reward: 14.000000, loss: 0.005161, epsilon: 0.050000, episode:  309\n",
      "frames: 562000, reward: 14.000000, loss: 0.001734, epsilon: 0.050000, episode:  309\n",
      "frames: 563000, reward: 14.000000, loss: 0.002644, epsilon: 0.050000, episode:  309\n",
      "frames: 564000, reward: 14.000000, loss: 0.004699, epsilon: 0.050000, episode:  310\n",
      "frames: 565000, reward: 14.000000, loss: 0.008930, epsilon: 0.050000, episode:  310\n",
      "frames: 566000, reward: 14.200000, loss: 0.005180, epsilon: 0.050000, episode:  311\n",
      "frames: 567000, reward: 14.200000, loss: 0.003374, epsilon: 0.050000, episode:  311\n",
      "frames: 568000, reward: 13.000000, loss: 0.005333, epsilon: 0.050000, episode:  312\n",
      "frames: 569000, reward: 13.000000, loss: 0.002424, epsilon: 0.050000, episode:  312\n",
      "frames: 570000, reward: 13.300000, loss: 0.001841, epsilon: 0.050000, episode:  313\n",
      "frames: 571000, reward: 13.300000, loss: 0.003871, epsilon: 0.050000, episode:  313\n",
      "frames: 572000, reward: 12.900000, loss: 0.005622, epsilon: 0.050000, episode:  314\n",
      "frames: 573000, reward: 12.900000, loss: 0.005992, epsilon: 0.050000, episode:  314\n",
      "frames: 574000, reward: 12.900000, loss: 0.003615, epsilon: 0.050000, episode:  314\n",
      "frames: 575000, reward: 12.300000, loss: 0.003760, epsilon: 0.050000, episode:  315\n",
      "frames: 576000, reward: 12.300000, loss: 0.003250, epsilon: 0.050000, episode:  315\n",
      "frames: 577000, reward: 11.700000, loss: 0.008767, epsilon: 0.050000, episode:  316\n",
      "frames: 578000, reward: 11.700000, loss: 0.003157, epsilon: 0.050000, episode:  316\n",
      "frames: 579000, reward: 11.600000, loss: 0.002640, epsilon: 0.050000, episode:  317\n",
      "frames: 580000, reward: 11.600000, loss: 0.002399, epsilon: 0.050000, episode:  317\n",
      "frames: 581000, reward: 11.600000, loss: 0.002780, epsilon: 0.050000, episode:  317\n",
      "frames: 582000, reward: 11.300000, loss: 0.003206, epsilon: 0.050000, episode:  318\n",
      "frames: 583000, reward: 12.200000, loss: 0.006013, epsilon: 0.050000, episode:  319\n",
      "frames: 584000, reward: 12.200000, loss: 0.002663, epsilon: 0.050000, episode:  319\n",
      "frames: 585000, reward: 12.200000, loss: 0.002200, epsilon: 0.050000, episode:  319\n",
      "frames: 586000, reward: 11.900000, loss: 0.003174, epsilon: 0.050000, episode:  320\n",
      "frames: 587000, reward: 11.900000, loss: 0.004024, epsilon: 0.050000, episode:  320\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 588000, reward: 11.400000, loss: 0.002070, epsilon: 0.050000, episode:  321\n",
      "frames: 589000, reward: 11.400000, loss: 0.002823, epsilon: 0.050000, episode:  321\n",
      "frames: 590000, reward: 12.100000, loss: 0.002488, epsilon: 0.050000, episode:  322\n",
      "frames: 591000, reward: 12.100000, loss: 0.001981, epsilon: 0.050000, episode:  322\n",
      "frames: 592000, reward: 12.100000, loss: 0.003493, epsilon: 0.050000, episode:  322\n",
      "frames: 593000, reward: 11.700000, loss: 0.003350, epsilon: 0.050000, episode:  323\n",
      "frames: 594000, reward: 12.300000, loss: 0.003114, epsilon: 0.050000, episode:  324\n",
      "frames: 595000, reward: 12.300000, loss: 0.001970, epsilon: 0.050000, episode:  324\n",
      "frames: 596000, reward: 12.800000, loss: 0.006403, epsilon: 0.050000, episode:  325\n",
      "frames: 597000, reward: 12.800000, loss: 0.002783, epsilon: 0.050000, episode:  325\n",
      "frames: 598000, reward: 12.800000, loss: 0.004077, epsilon: 0.050000, episode:  325\n",
      "frames: 599000, reward: 13.200000, loss: 0.006125, epsilon: 0.050000, episode:  326\n",
      "frames: 600000, reward: 13.200000, loss: 0.003202, epsilon: 0.050000, episode:  326\n",
      "frames: 601000, reward: 13.100000, loss: 0.003550, epsilon: 0.050000, episode:  327\n",
      "frames: 602000, reward: 13.100000, loss: 0.004488, epsilon: 0.050000, episode:  327\n",
      "frames: 603000, reward: 13.300000, loss: 0.003670, epsilon: 0.050000, episode:  328\n",
      "frames: 604000, reward: 13.300000, loss: 0.002966, epsilon: 0.050000, episode:  328\n",
      "frames: 605000, reward: 11.800000, loss: 0.004361, epsilon: 0.050000, episode:  329\n",
      "frames: 606000, reward: 11.800000, loss: 0.001549, epsilon: 0.050000, episode:  329\n",
      "frames: 607000, reward: 12.300000, loss: 0.003826, epsilon: 0.050000, episode:  330\n",
      "frames: 608000, reward: 12.300000, loss: 0.002080, epsilon: 0.050000, episode:  330\n",
      "frames: 609000, reward: 12.300000, loss: 0.002990, epsilon: 0.050000, episode:  330\n",
      "frames: 610000, reward: 12.300000, loss: 0.005577, epsilon: 0.050000, episode:  331\n",
      "frames: 611000, reward: 12.300000, loss: 0.002862, epsilon: 0.050000, episode:  331\n",
      "frames: 612000, reward: 12.500000, loss: 0.004980, epsilon: 0.050000, episode:  332\n",
      "frames: 613000, reward: 12.500000, loss: 0.007164, epsilon: 0.050000, episode:  332\n",
      "frames: 614000, reward: 12.000000, loss: 0.004300, epsilon: 0.050000, episode:  333\n",
      "frames: 615000, reward: 12.000000, loss: 0.004539, epsilon: 0.050000, episode:  333\n",
      "frames: 616000, reward: 11.500000, loss: 0.004464, epsilon: 0.050000, episode:  334\n",
      "frames: 617000, reward: 11.500000, loss: 0.005949, epsilon: 0.050000, episode:  334\n",
      "frames: 618000, reward: 11.500000, loss: 0.005583, epsilon: 0.050000, episode:  334\n",
      "frames: 619000, reward: 11.000000, loss: 0.003963, epsilon: 0.050000, episode:  335\n",
      "frames: 620000, reward: 11.800000, loss: 0.003199, epsilon: 0.050000, episode:  336\n",
      "frames: 621000, reward: 11.800000, loss: 0.002083, epsilon: 0.050000, episode:  336\n",
      "frames: 622000, reward: 11.800000, loss: 0.002493, epsilon: 0.050000, episode:  336\n",
      "frames: 623000, reward: 10.500000, loss: 0.004148, epsilon: 0.050000, episode:  337\n",
      "frames: 624000, reward: 10.500000, loss: 0.004712, epsilon: 0.050000, episode:  337\n",
      "frames: 625000, reward: 9.900000, loss: 0.003125, epsilon: 0.050000, episode:  338\n",
      "frames: 626000, reward: 9.900000, loss: 0.008460, epsilon: 0.050000, episode:  338\n",
      "frames: 627000, reward: 9.900000, loss: 0.012977, epsilon: 0.050000, episode:  338\n",
      "frames: 628000, reward: 10.600000, loss: 0.004257, epsilon: 0.050000, episode:  339\n",
      "frames: 629000, reward: 10.600000, loss: 0.004101, epsilon: 0.050000, episode:  339\n",
      "frames: 630000, reward: 10.500000, loss: 0.004496, epsilon: 0.050000, episode:  340\n",
      "frames: 631000, reward: 10.500000, loss: 0.006106, epsilon: 0.050000, episode:  340\n",
      "frames: 632000, reward: 10.900000, loss: 0.004668, epsilon: 0.050000, episode:  341\n",
      "frames: 633000, reward: 10.900000, loss: 0.006766, epsilon: 0.050000, episode:  341\n",
      "frames: 634000, reward: 10.900000, loss: 0.002451, epsilon: 0.050000, episode:  342\n",
      "frames: 635000, reward: 10.900000, loss: 0.001933, epsilon: 0.050000, episode:  342\n",
      "frames: 636000, reward: 11.800000, loss: 0.007421, epsilon: 0.050000, episode:  343\n",
      "frames: 637000, reward: 11.800000, loss: 0.003743, epsilon: 0.050000, episode:  343\n",
      "frames: 638000, reward: 12.000000, loss: 0.001552, epsilon: 0.050000, episode:  344\n",
      "frames: 639000, reward: 12.000000, loss: 0.003951, epsilon: 0.050000, episode:  344\n",
      "frames: 640000, reward: 12.900000, loss: 0.004860, epsilon: 0.050000, episode:  345\n",
      "frames: 641000, reward: 12.900000, loss: 0.001916, epsilon: 0.050000, episode:  345\n",
      "frames: 642000, reward: 12.700000, loss: 0.013240, epsilon: 0.050000, episode:  346\n",
      "frames: 643000, reward: 12.700000, loss: 0.004675, epsilon: 0.050000, episode:  346\n",
      "frames: 644000, reward: 13.800000, loss: 0.001945, epsilon: 0.050000, episode:  347\n",
      "frames: 645000, reward: 13.800000, loss: 0.005272, epsilon: 0.050000, episode:  347\n",
      "frames: 646000, reward: 14.500000, loss: 0.001491, epsilon: 0.050000, episode:  348\n",
      "frames: 647000, reward: 14.500000, loss: 0.002338, epsilon: 0.050000, episode:  348\n",
      "frames: 648000, reward: 14.100000, loss: 0.002657, epsilon: 0.050000, episode:  349\n",
      "frames: 649000, reward: 14.100000, loss: 0.001703, epsilon: 0.050000, episode:  349\n",
      "frames: 650000, reward: 14.500000, loss: 0.003099, epsilon: 0.050000, episode:  350\n",
      "frames: 651000, reward: 14.500000, loss: 0.004260, epsilon: 0.050000, episode:  350\n",
      "frames: 652000, reward: 14.500000, loss: 0.003812, epsilon: 0.050000, episode:  350\n",
      "frames: 653000, reward: 13.800000, loss: 0.005389, epsilon: 0.050000, episode:  351\n",
      "frames: 654000, reward: 13.800000, loss: 0.003579, epsilon: 0.050000, episode:  351\n",
      "frames: 655000, reward: 14.000000, loss: 0.006370, epsilon: 0.050000, episode:  352\n",
      "frames: 656000, reward: 14.000000, loss: 0.005683, epsilon: 0.050000, episode:  352\n",
      "frames: 657000, reward: 13.600000, loss: 0.007564, epsilon: 0.050000, episode:  353\n",
      "frames: 658000, reward: 13.600000, loss: 0.002054, epsilon: 0.050000, episode:  353\n",
      "frames: 659000, reward: 13.400000, loss: 0.002324, epsilon: 0.050000, episode:  354\n",
      "frames: 660000, reward: 13.400000, loss: 0.002496, epsilon: 0.050000, episode:  354\n",
      "frames: 661000, reward: 13.400000, loss: 0.004935, epsilon: 0.050000, episode:  354\n",
      "frames: 662000, reward: 12.400000, loss: 0.007861, epsilon: 0.050000, episode:  355\n",
      "frames: 663000, reward: 12.400000, loss: 0.002743, epsilon: 0.050000, episode:  355\n",
      "frames: 664000, reward: 12.400000, loss: 0.003260, epsilon: 0.050000, episode:  355\n",
      "frames: 665000, reward: 11.000000, loss: 0.004895, epsilon: 0.050000, episode:  356\n",
      "frames: 666000, reward: 11.000000, loss: 0.004499, epsilon: 0.050000, episode:  356\n",
      "frames: 667000, reward: 11.200000, loss: 0.007555, epsilon: 0.050000, episode:  357\n",
      "frames: 668000, reward: 11.200000, loss: 0.004246, epsilon: 0.050000, episode:  357\n",
      "frames: 669000, reward: 10.200000, loss: 0.002557, epsilon: 0.050000, episode:  358\n",
      "frames: 670000, reward: 10.200000, loss: 0.005413, epsilon: 0.050000, episode:  358\n",
      "frames: 671000, reward: 11.700000, loss: 0.002824, epsilon: 0.050000, episode:  359\n",
      "frames: 672000, reward: 11.700000, loss: 0.001888, epsilon: 0.050000, episode:  359\n",
      "frames: 673000, reward: 11.900000, loss: 0.002795, epsilon: 0.050000, episode:  360\n",
      "frames: 674000, reward: 11.900000, loss: 0.001876, epsilon: 0.050000, episode:  360\n",
      "frames: 675000, reward: 12.900000, loss: 0.004523, epsilon: 0.050000, episode:  361\n",
      "frames: 676000, reward: 12.900000, loss: 0.003537, epsilon: 0.050000, episode:  361\n",
      "frames: 677000, reward: 11.900000, loss: 0.016604, epsilon: 0.050000, episode:  362\n",
      "frames: 678000, reward: 11.900000, loss: 0.006508, epsilon: 0.050000, episode:  362\n",
      "frames: 679000, reward: 11.900000, loss: 0.001520, epsilon: 0.050000, episode:  362\n",
      "frames: 680000, reward: 11.300000, loss: 0.006432, epsilon: 0.050000, episode:  363\n",
      "frames: 681000, reward: 11.300000, loss: 0.013989, epsilon: 0.050000, episode:  363\n",
      "frames: 682000, reward: 11.400000, loss: 0.001962, epsilon: 0.050000, episode:  364\n",
      "frames: 683000, reward: 11.400000, loss: 0.006772, epsilon: 0.050000, episode:  364\n",
      "frames: 684000, reward: 12.000000, loss: 0.001740, epsilon: 0.050000, episode:  365\n",
      "frames: 685000, reward: 12.000000, loss: 0.006663, epsilon: 0.050000, episode:  365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 686000, reward: 12.700000, loss: 0.013262, epsilon: 0.050000, episode:  366\n",
      "frames: 687000, reward: 12.700000, loss: 0.003589, epsilon: 0.050000, episode:  366\n",
      "frames: 688000, reward: 12.500000, loss: 0.005112, epsilon: 0.050000, episode:  367\n",
      "frames: 689000, reward: 12.500000, loss: 0.002149, epsilon: 0.050000, episode:  367\n",
      "frames: 690000, reward: 13.300000, loss: 0.001669, epsilon: 0.050000, episode:  368\n",
      "frames: 691000, reward: 13.300000, loss: 0.001755, epsilon: 0.050000, episode:  368\n",
      "frames: 692000, reward: 13.200000, loss: 0.006534, epsilon: 0.050000, episode:  369\n",
      "frames: 693000, reward: 13.200000, loss: 0.004343, epsilon: 0.050000, episode:  369\n",
      "frames: 694000, reward: 12.800000, loss: 0.004090, epsilon: 0.050000, episode:  370\n",
      "frames: 695000, reward: 12.800000, loss: 0.006033, epsilon: 0.050000, episode:  370\n",
      "frames: 696000, reward: 11.900000, loss: 0.002395, epsilon: 0.050000, episode:  371\n",
      "frames: 697000, reward: 11.900000, loss: 0.002055, epsilon: 0.050000, episode:  371\n",
      "frames: 698000, reward: 11.900000, loss: 0.006505, epsilon: 0.050000, episode:  371\n",
      "frames: 699000, reward: 12.800000, loss: 0.005015, epsilon: 0.050000, episode:  372\n",
      "frames: 700000, reward: 14.100000, loss: 0.003381, epsilon: 0.050000, episode:  373\n",
      "frames: 701000, reward: 14.100000, loss: 0.002979, epsilon: 0.050000, episode:  373\n",
      "frames: 702000, reward: 14.100000, loss: 0.006446, epsilon: 0.050000, episode:  373\n",
      "frames: 703000, reward: 13.900000, loss: 0.002931, epsilon: 0.050000, episode:  374\n",
      "frames: 704000, reward: 13.900000, loss: 0.001593, epsilon: 0.050000, episode:  374\n",
      "frames: 705000, reward: 13.900000, loss: 0.008251, epsilon: 0.050000, episode:  374\n",
      "frames: 706000, reward: 12.700000, loss: 0.003460, epsilon: 0.050000, episode:  375\n",
      "frames: 707000, reward: 12.700000, loss: 0.004000, epsilon: 0.050000, episode:  375\n",
      "frames: 708000, reward: 13.000000, loss: 0.003748, epsilon: 0.050000, episode:  376\n",
      "frames: 709000, reward: 13.000000, loss: 0.002499, epsilon: 0.050000, episode:  376\n",
      "frames: 710000, reward: 13.000000, loss: 0.005831, epsilon: 0.050000, episode:  376\n",
      "frames: 711000, reward: 11.900000, loss: 0.005904, epsilon: 0.050000, episode:  377\n",
      "frames: 712000, reward: 11.900000, loss: 0.004192, epsilon: 0.050000, episode:  377\n",
      "frames: 713000, reward: 10.300000, loss: 0.006425, epsilon: 0.050000, episode:  378\n",
      "frames: 714000, reward: 10.300000, loss: 0.003397, epsilon: 0.050000, episode:  378\n",
      "frames: 715000, reward: 10.300000, loss: 0.005000, epsilon: 0.050000, episode:  378\n",
      "frames: 716000, reward: 9.100000, loss: 0.003751, epsilon: 0.050000, episode:  379\n",
      "frames: 717000, reward: 9.100000, loss: 0.004580, epsilon: 0.050000, episode:  379\n",
      "frames: 718000, reward: 8.900000, loss: 0.005108, epsilon: 0.050000, episode:  380\n",
      "frames: 719000, reward: 8.900000, loss: 0.011194, epsilon: 0.050000, episode:  380\n",
      "frames: 720000, reward: 8.900000, loss: 0.004934, epsilon: 0.050000, episode:  380\n",
      "frames: 721000, reward: 9.100000, loss: 0.004849, epsilon: 0.050000, episode:  381\n",
      "frames: 722000, reward: 9.100000, loss: 0.013543, epsilon: 0.050000, episode:  381\n",
      "frames: 723000, reward: 8.300000, loss: 0.004437, epsilon: 0.050000, episode:  382\n",
      "frames: 724000, reward: 8.300000, loss: 0.001230, epsilon: 0.050000, episode:  382\n",
      "frames: 725000, reward: 8.300000, loss: 0.008517, epsilon: 0.050000, episode:  382\n",
      "frames: 726000, reward: 7.500000, loss: 0.011976, epsilon: 0.050000, episode:  383\n",
      "frames: 727000, reward: 7.500000, loss: 0.003772, epsilon: 0.050000, episode:  383\n",
      "frames: 728000, reward: 7.200000, loss: 0.002940, epsilon: 0.050000, episode:  384\n",
      "frames: 729000, reward: 7.200000, loss: 0.003612, epsilon: 0.050000, episode:  384\n",
      "frames: 730000, reward: 9.000000, loss: 0.003040, epsilon: 0.050000, episode:  385\n",
      "frames: 731000, reward: 9.000000, loss: 0.003127, epsilon: 0.050000, episode:  385\n",
      "frames: 732000, reward: 8.800000, loss: 0.003885, epsilon: 0.050000, episode:  386\n",
      "frames: 733000, reward: 8.800000, loss: 0.002728, epsilon: 0.050000, episode:  386\n",
      "frames: 734000, reward: 9.500000, loss: 0.004530, epsilon: 0.050000, episode:  387\n",
      "frames: 735000, reward: 9.500000, loss: 0.004858, epsilon: 0.050000, episode:  387\n",
      "frames: 736000, reward: 10.800000, loss: 0.005801, epsilon: 0.050000, episode:  388\n",
      "frames: 737000, reward: 10.800000, loss: 0.003047, epsilon: 0.050000, episode:  388\n",
      "frames: 738000, reward: 11.800000, loss: 0.001172, epsilon: 0.050000, episode:  389\n",
      "frames: 739000, reward: 11.800000, loss: 0.004671, epsilon: 0.050000, episode:  389\n",
      "frames: 740000, reward: 12.200000, loss: 0.003640, epsilon: 0.050000, episode:  390\n",
      "frames: 741000, reward: 12.200000, loss: 0.006269, epsilon: 0.050000, episode:  390\n",
      "frames: 742000, reward: 12.600000, loss: 0.003510, epsilon: 0.050000, episode:  391\n",
      "frames: 743000, reward: 12.600000, loss: 0.006149, epsilon: 0.050000, episode:  391\n",
      "frames: 744000, reward: 13.700000, loss: 0.002150, epsilon: 0.050000, episode:  392\n",
      "frames: 745000, reward: 13.700000, loss: 0.003094, epsilon: 0.050000, episode:  392\n",
      "frames: 746000, reward: 14.200000, loss: 0.003410, epsilon: 0.050000, episode:  393\n",
      "frames: 747000, reward: 14.200000, loss: 0.002513, epsilon: 0.050000, episode:  393\n",
      "frames: 748000, reward: 15.200000, loss: 0.002758, epsilon: 0.050000, episode:  394\n",
      "frames: 749000, reward: 15.200000, loss: 0.003483, epsilon: 0.050000, episode:  394\n",
      "frames: 750000, reward: 14.900000, loss: 0.002084, epsilon: 0.050000, episode:  395\n",
      "frames: 751000, reward: 14.900000, loss: 0.001480, epsilon: 0.050000, episode:  395\n",
      "frames: 752000, reward: 15.300000, loss: 0.003257, epsilon: 0.050000, episode:  396\n",
      "frames: 753000, reward: 15.300000, loss: 0.001777, epsilon: 0.050000, episode:  396\n",
      "frames: 754000, reward: 15.300000, loss: 0.003351, epsilon: 0.050000, episode:  396\n",
      "frames: 755000, reward: 15.100000, loss: 0.003459, epsilon: 0.050000, episode:  397\n",
      "frames: 756000, reward: 15.100000, loss: 0.004224, epsilon: 0.050000, episode:  397\n",
      "frames: 757000, reward: 14.900000, loss: 0.007921, epsilon: 0.050000, episode:  398\n",
      "frames: 758000, reward: 14.900000, loss: 0.001979, epsilon: 0.050000, episode:  398\n",
      "frames: 759000, reward: 14.800000, loss: 0.002553, epsilon: 0.050000, episode:  399\n",
      "frames: 760000, reward: 14.800000, loss: 0.006769, epsilon: 0.050000, episode:  399\n",
      "frames: 761000, reward: 14.500000, loss: 0.005826, epsilon: 0.050000, episode:  400\n",
      "frames: 762000, reward: 14.500000, loss: 0.002541, epsilon: 0.050000, episode:  400\n",
      "frames: 763000, reward: 14.000000, loss: 0.004926, epsilon: 0.050000, episode:  401\n",
      "frames: 764000, reward: 14.000000, loss: 0.008208, epsilon: 0.050000, episode:  401\n",
      "frames: 765000, reward: 13.600000, loss: 0.002703, epsilon: 0.050000, episode:  402\n",
      "frames: 766000, reward: 13.600000, loss: 0.003450, epsilon: 0.050000, episode:  402\n",
      "frames: 767000, reward: 13.700000, loss: 0.001691, epsilon: 0.050000, episode:  403\n",
      "frames: 768000, reward: 13.700000, loss: 0.022533, epsilon: 0.050000, episode:  403\n",
      "frames: 769000, reward: 13.700000, loss: 0.003179, epsilon: 0.050000, episode:  403\n",
      "frames: 770000, reward: 12.600000, loss: 0.003322, epsilon: 0.050000, episode:  404\n",
      "frames: 771000, reward: 12.600000, loss: 0.002630, epsilon: 0.050000, episode:  404\n",
      "frames: 772000, reward: 12.600000, loss: 0.002720, epsilon: 0.050000, episode:  405\n",
      "frames: 773000, reward: 13.000000, loss: 0.003360, epsilon: 0.050000, episode:  406\n",
      "frames: 774000, reward: 13.000000, loss: 0.003238, epsilon: 0.050000, episode:  406\n",
      "frames: 775000, reward: 14.200000, loss: 0.002837, epsilon: 0.050000, episode:  407\n",
      "frames: 776000, reward: 14.200000, loss: 0.004982, epsilon: 0.050000, episode:  407\n",
      "frames: 777000, reward: 14.200000, loss: 0.004870, epsilon: 0.050000, episode:  408\n",
      "frames: 778000, reward: 14.200000, loss: 0.001830, epsilon: 0.050000, episode:  408\n",
      "frames: 779000, reward: 14.500000, loss: 0.001590, epsilon: 0.050000, episode:  409\n",
      "frames: 780000, reward: 14.500000, loss: 0.003371, epsilon: 0.050000, episode:  409\n",
      "frames: 781000, reward: 14.900000, loss: 0.001367, epsilon: 0.050000, episode:  410\n",
      "frames: 782000, reward: 14.900000, loss: 0.002215, epsilon: 0.050000, episode:  410\n",
      "frames: 783000, reward: 15.300000, loss: 0.001211, epsilon: 0.050000, episode:  411\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 784000, reward: 15.300000, loss: 0.005930, epsilon: 0.050000, episode:  411\n",
      "frames: 785000, reward: 15.300000, loss: 0.003688, epsilon: 0.050000, episode:  412\n",
      "frames: 786000, reward: 15.300000, loss: 0.005271, epsilon: 0.050000, episode:  412\n",
      "frames: 787000, reward: 15.300000, loss: 0.003653, epsilon: 0.050000, episode:  413\n",
      "frames: 788000, reward: 15.300000, loss: 0.001813, epsilon: 0.050000, episode:  413\n",
      "frames: 789000, reward: 15.800000, loss: 0.002778, epsilon: 0.050000, episode:  414\n",
      "frames: 790000, reward: 15.800000, loss: 0.003190, epsilon: 0.050000, episode:  414\n",
      "frames: 791000, reward: 15.500000, loss: 0.004202, epsilon: 0.050000, episode:  415\n",
      "frames: 792000, reward: 15.500000, loss: 0.001370, epsilon: 0.050000, episode:  415\n",
      "frames: 793000, reward: 14.300000, loss: 0.002783, epsilon: 0.050000, episode:  416\n",
      "frames: 794000, reward: 14.300000, loss: 0.005818, epsilon: 0.050000, episode:  416\n",
      "frames: 795000, reward: 14.100000, loss: 0.004066, epsilon: 0.050000, episode:  417\n",
      "frames: 796000, reward: 14.100000, loss: 0.003288, epsilon: 0.050000, episode:  417\n",
      "frames: 797000, reward: 14.800000, loss: 0.001890, epsilon: 0.050000, episode:  418\n",
      "frames: 798000, reward: 14.800000, loss: 0.002120, epsilon: 0.050000, episode:  418\n",
      "frames: 799000, reward: 14.800000, loss: 0.003786, epsilon: 0.050000, episode:  419\n",
      "frames: 800000, reward: 14.800000, loss: 0.001102, epsilon: 0.050000, episode:  419\n",
      "frames: 801000, reward: 14.000000, loss: 0.003796, epsilon: 0.050000, episode:  420\n",
      "frames: 802000, reward: 14.000000, loss: 0.009564, epsilon: 0.050000, episode:  420\n",
      "frames: 803000, reward: 14.000000, loss: 0.003708, epsilon: 0.050000, episode:  420\n",
      "frames: 804000, reward: 12.900000, loss: 0.002839, epsilon: 0.050000, episode:  421\n",
      "frames: 805000, reward: 12.900000, loss: 0.002119, epsilon: 0.050000, episode:  421\n",
      "frames: 806000, reward: 12.500000, loss: 0.001886, epsilon: 0.050000, episode:  422\n",
      "frames: 807000, reward: 12.500000, loss: 0.005183, epsilon: 0.050000, episode:  422\n",
      "frames: 808000, reward: 12.500000, loss: 0.005281, epsilon: 0.050000, episode:  423\n",
      "frames: 809000, reward: 12.500000, loss: 0.002836, epsilon: 0.050000, episode:  423\n",
      "frames: 810000, reward: 12.100000, loss: 0.005693, epsilon: 0.050000, episode:  424\n",
      "frames: 811000, reward: 12.100000, loss: 0.002112, epsilon: 0.050000, episode:  424\n",
      "frames: 812000, reward: 12.100000, loss: 0.003815, epsilon: 0.050000, episode:  424\n",
      "frames: 813000, reward: 11.500000, loss: 0.003822, epsilon: 0.050000, episode:  425\n",
      "frames: 814000, reward: 11.500000, loss: 0.003786, epsilon: 0.050000, episode:  425\n",
      "frames: 815000, reward: 12.800000, loss: 0.002389, epsilon: 0.050000, episode:  426\n",
      "frames: 816000, reward: 12.800000, loss: 0.006860, epsilon: 0.050000, episode:  426\n",
      "frames: 817000, reward: 12.100000, loss: 0.004031, epsilon: 0.050000, episode:  427\n",
      "frames: 818000, reward: 12.100000, loss: 0.007472, epsilon: 0.050000, episode:  427\n",
      "frames: 819000, reward: 11.600000, loss: 0.005073, epsilon: 0.050000, episode:  428\n",
      "frames: 820000, reward: 11.600000, loss: 0.005146, epsilon: 0.050000, episode:  428\n",
      "frames: 821000, reward: 11.400000, loss: 0.005776, epsilon: 0.050000, episode:  429\n",
      "frames: 822000, reward: 11.400000, loss: 0.007767, epsilon: 0.050000, episode:  429\n",
      "frames: 823000, reward: 12.000000, loss: 0.018277, epsilon: 0.050000, episode:  430\n",
      "frames: 824000, reward: 12.000000, loss: 0.003227, epsilon: 0.050000, episode:  430\n",
      "frames: 825000, reward: 12.800000, loss: 0.009925, epsilon: 0.050000, episode:  431\n",
      "frames: 826000, reward: 12.800000, loss: 0.005208, epsilon: 0.050000, episode:  431\n",
      "frames: 827000, reward: 13.700000, loss: 0.003596, epsilon: 0.050000, episode:  432\n",
      "frames: 828000, reward: 13.700000, loss: 0.003068, epsilon: 0.050000, episode:  432\n",
      "frames: 829000, reward: 13.300000, loss: 0.003422, epsilon: 0.050000, episode:  433\n",
      "frames: 830000, reward: 13.300000, loss: 0.003936, epsilon: 0.050000, episode:  433\n",
      "frames: 831000, reward: 14.200000, loss: 0.002346, epsilon: 0.050000, episode:  434\n",
      "frames: 832000, reward: 14.200000, loss: 0.010142, epsilon: 0.050000, episode:  434\n",
      "frames: 833000, reward: 15.200000, loss: 0.002244, epsilon: 0.050000, episode:  435\n",
      "frames: 834000, reward: 15.200000, loss: 0.002677, epsilon: 0.050000, episode:  435\n",
      "frames: 835000, reward: 14.800000, loss: 0.002685, epsilon: 0.050000, episode:  436\n",
      "frames: 836000, reward: 14.800000, loss: 0.002698, epsilon: 0.050000, episode:  436\n",
      "frames: 837000, reward: 15.000000, loss: 0.019858, epsilon: 0.050000, episode:  437\n",
      "frames: 838000, reward: 15.000000, loss: 0.007048, epsilon: 0.050000, episode:  437\n",
      "frames: 839000, reward: 15.100000, loss: 0.004613, epsilon: 0.050000, episode:  438\n",
      "frames: 840000, reward: 15.100000, loss: 0.012858, epsilon: 0.050000, episode:  438\n",
      "frames: 841000, reward: 15.100000, loss: 0.002194, epsilon: 0.050000, episode:  439\n",
      "frames: 842000, reward: 15.100000, loss: 0.002559, epsilon: 0.050000, episode:  439\n",
      "frames: 843000, reward: 14.600000, loss: 0.001924, epsilon: 0.050000, episode:  440\n",
      "frames: 844000, reward: 14.600000, loss: 0.003627, epsilon: 0.050000, episode:  440\n",
      "frames: 845000, reward: 14.600000, loss: 0.004548, epsilon: 0.050000, episode:  440\n",
      "frames: 846000, reward: 14.600000, loss: 0.004621, epsilon: 0.050000, episode:  441\n",
      "frames: 847000, reward: 14.600000, loss: 0.005549, epsilon: 0.050000, episode:  441\n",
      "frames: 848000, reward: 14.000000, loss: 0.005963, epsilon: 0.050000, episode:  442\n",
      "frames: 849000, reward: 14.000000, loss: 0.005065, epsilon: 0.050000, episode:  442\n",
      "frames: 850000, reward: 14.000000, loss: 0.012015, epsilon: 0.050000, episode:  442\n",
      "frames: 851000, reward: 13.400000, loss: 0.003502, epsilon: 0.050000, episode:  443\n",
      "frames: 852000, reward: 13.400000, loss: 0.002603, epsilon: 0.050000, episode:  443\n",
      "frames: 853000, reward: 12.500000, loss: 0.003947, epsilon: 0.050000, episode:  444\n",
      "frames: 854000, reward: 12.500000, loss: 0.008466, epsilon: 0.050000, episode:  444\n",
      "frames: 855000, reward: 11.900000, loss: 0.004562, epsilon: 0.050000, episode:  445\n",
      "frames: 856000, reward: 11.900000, loss: 0.002406, epsilon: 0.050000, episode:  445\n",
      "frames: 857000, reward: 12.000000, loss: 0.003552, epsilon: 0.050000, episode:  446\n",
      "frames: 858000, reward: 12.000000, loss: 0.002255, epsilon: 0.050000, episode:  446\n",
      "frames: 859000, reward: 12.400000, loss: 0.002330, epsilon: 0.050000, episode:  447\n",
      "frames: 860000, reward: 12.400000, loss: 0.003306, epsilon: 0.050000, episode:  447\n",
      "frames: 861000, reward: 12.300000, loss: 0.002213, epsilon: 0.050000, episode:  448\n",
      "frames: 862000, reward: 12.300000, loss: 0.003722, epsilon: 0.050000, episode:  448\n",
      "frames: 863000, reward: 11.900000, loss: 0.004173, epsilon: 0.050000, episode:  449\n",
      "frames: 864000, reward: 11.900000, loss: 0.001845, epsilon: 0.050000, episode:  449\n",
      "frames: 865000, reward: 12.500000, loss: 0.002731, epsilon: 0.050000, episode:  450\n",
      "frames: 866000, reward: 12.500000, loss: 0.002398, epsilon: 0.050000, episode:  450\n",
      "frames: 867000, reward: 13.200000, loss: 0.002803, epsilon: 0.050000, episode:  451\n",
      "frames: 868000, reward: 13.200000, loss: 0.004009, epsilon: 0.050000, episode:  451\n",
      "frames: 869000, reward: 13.200000, loss: 0.005148, epsilon: 0.050000, episode:  452\n",
      "frames: 870000, reward: 13.200000, loss: 0.003827, epsilon: 0.050000, episode:  452\n",
      "frames: 871000, reward: 13.700000, loss: 0.003327, epsilon: 0.050000, episode:  453\n",
      "frames: 872000, reward: 13.700000, loss: 0.002077, epsilon: 0.050000, episode:  453\n",
      "frames: 873000, reward: 14.600000, loss: 0.004033, epsilon: 0.050000, episode:  454\n",
      "frames: 874000, reward: 14.600000, loss: 0.002994, epsilon: 0.050000, episode:  454\n",
      "frames: 875000, reward: 15.200000, loss: 0.002673, epsilon: 0.050000, episode:  455\n",
      "frames: 876000, reward: 15.200000, loss: 0.005663, epsilon: 0.050000, episode:  455\n",
      "frames: 877000, reward: 14.700000, loss: 0.002320, epsilon: 0.050000, episode:  456\n",
      "frames: 878000, reward: 14.700000, loss: 0.003992, epsilon: 0.050000, episode:  456\n",
      "frames: 879000, reward: 14.800000, loss: 0.003739, epsilon: 0.050000, episode:  457\n",
      "frames: 880000, reward: 14.800000, loss: 0.003750, epsilon: 0.050000, episode:  457\n",
      "frames: 881000, reward: 14.800000, loss: 0.004355, epsilon: 0.050000, episode:  457\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 882000, reward: 14.200000, loss: 0.001250, epsilon: 0.050000, episode:  458\n",
      "frames: 883000, reward: 14.200000, loss: 0.007638, epsilon: 0.050000, episode:  458\n",
      "frames: 884000, reward: 14.400000, loss: 0.009678, epsilon: 0.050000, episode:  459\n",
      "frames: 885000, reward: 14.400000, loss: 0.005562, epsilon: 0.050000, episode:  459\n",
      "frames: 886000, reward: 14.300000, loss: 0.003816, epsilon: 0.050000, episode:  460\n",
      "frames: 887000, reward: 14.300000, loss: 0.004365, epsilon: 0.050000, episode:  460\n",
      "frames: 888000, reward: 14.000000, loss: 0.002744, epsilon: 0.050000, episode:  461\n",
      "frames: 889000, reward: 14.800000, loss: 0.003915, epsilon: 0.050000, episode:  462\n",
      "frames: 890000, reward: 14.800000, loss: 0.008042, epsilon: 0.050000, episode:  462\n",
      "frames: 891000, reward: 14.800000, loss: 0.006498, epsilon: 0.050000, episode:  462\n",
      "frames: 892000, reward: 14.700000, loss: 0.004728, epsilon: 0.050000, episode:  463\n",
      "frames: 893000, reward: 14.700000, loss: 0.007257, epsilon: 0.050000, episode:  463\n",
      "frames: 894000, reward: 14.400000, loss: 0.001448, epsilon: 0.050000, episode:  464\n",
      "frames: 895000, reward: 14.400000, loss: 0.001285, epsilon: 0.050000, episode:  464\n",
      "frames: 896000, reward: 13.400000, loss: 0.003253, epsilon: 0.050000, episode:  465\n",
      "frames: 897000, reward: 13.400000, loss: 0.002205, epsilon: 0.050000, episode:  465\n",
      "frames: 898000, reward: 13.800000, loss: 0.003359, epsilon: 0.050000, episode:  466\n",
      "frames: 899000, reward: 13.800000, loss: 0.003532, epsilon: 0.050000, episode:  466\n",
      "frames: 900000, reward: 13.800000, loss: 0.001767, epsilon: 0.050000, episode:  467\n",
      "frames: 901000, reward: 13.800000, loss: 0.001797, epsilon: 0.050000, episode:  467\n",
      "frames: 902000, reward: 14.500000, loss: 0.002286, epsilon: 0.050000, episode:  468\n",
      "frames: 903000, reward: 14.500000, loss: 0.003187, epsilon: 0.050000, episode:  468\n",
      "frames: 904000, reward: 14.100000, loss: 0.003377, epsilon: 0.050000, episode:  469\n",
      "frames: 905000, reward: 14.100000, loss: 0.004311, epsilon: 0.050000, episode:  469\n",
      "frames: 906000, reward: 14.000000, loss: 0.003630, epsilon: 0.050000, episode:  470\n",
      "frames: 907000, reward: 14.000000, loss: 0.004132, epsilon: 0.050000, episode:  470\n",
      "frames: 908000, reward: 14.000000, loss: 0.006328, epsilon: 0.050000, episode:  470\n",
      "frames: 909000, reward: 14.100000, loss: 0.002455, epsilon: 0.050000, episode:  471\n",
      "frames: 910000, reward: 14.100000, loss: 0.004623, epsilon: 0.050000, episode:  471\n",
      "frames: 911000, reward: 13.000000, loss: 0.005231, epsilon: 0.050000, episode:  472\n",
      "frames: 912000, reward: 13.000000, loss: 0.003143, epsilon: 0.050000, episode:  472\n",
      "frames: 913000, reward: 12.900000, loss: 0.002076, epsilon: 0.050000, episode:  473\n",
      "frames: 914000, reward: 12.900000, loss: 0.005536, epsilon: 0.050000, episode:  473\n",
      "frames: 915000, reward: 12.900000, loss: 0.003451, epsilon: 0.050000, episode:  473\n",
      "frames: 916000, reward: 11.800000, loss: 0.003453, epsilon: 0.050000, episode:  474\n",
      "frames: 917000, reward: 11.800000, loss: 0.002682, epsilon: 0.050000, episode:  474\n",
      "frames: 918000, reward: 11.800000, loss: 0.002784, epsilon: 0.050000, episode:  474\n",
      "frames: 919000, reward: 11.400000, loss: 0.004373, epsilon: 0.050000, episode:  475\n",
      "frames: 920000, reward: 11.400000, loss: 0.005184, epsilon: 0.050000, episode:  476\n",
      "frames: 921000, reward: 11.400000, loss: 0.002715, epsilon: 0.050000, episode:  476\n",
      "frames: 922000, reward: 11.400000, loss: 0.004207, epsilon: 0.050000, episode:  476\n",
      "frames: 923000, reward: 11.000000, loss: 0.007164, epsilon: 0.050000, episode:  477\n",
      "frames: 924000, reward: 11.000000, loss: 0.002775, epsilon: 0.050000, episode:  477\n",
      "frames: 925000, reward: 10.400000, loss: 0.012318, epsilon: 0.050000, episode:  478\n",
      "frames: 926000, reward: 10.400000, loss: 0.003967, epsilon: 0.050000, episode:  478\n",
      "frames: 927000, reward: 10.400000, loss: 0.005391, epsilon: 0.050000, episode:  478\n",
      "frames: 928000, reward: 10.500000, loss: 0.003726, epsilon: 0.050000, episode:  479\n",
      "frames: 929000, reward: 10.800000, loss: 0.007272, epsilon: 0.050000, episode:  480\n",
      "frames: 930000, reward: 10.800000, loss: 0.001780, epsilon: 0.050000, episode:  480\n",
      "frames: 931000, reward: 10.800000, loss: 0.006036, epsilon: 0.050000, episode:  480\n",
      "frames: 932000, reward: 10.400000, loss: 0.004918, epsilon: 0.050000, episode:  481\n",
      "frames: 933000, reward: 10.400000, loss: 0.003078, epsilon: 0.050000, episode:  481\n",
      "frames: 934000, reward: 10.300000, loss: 0.004566, epsilon: 0.050000, episode:  482\n",
      "frames: 935000, reward: 10.300000, loss: 0.002873, epsilon: 0.050000, episode:  482\n",
      "frames: 936000, reward: 10.300000, loss: 0.002335, epsilon: 0.050000, episode:  483\n",
      "frames: 937000, reward: 10.300000, loss: 0.005304, epsilon: 0.050000, episode:  483\n",
      "frames: 938000, reward: 10.300000, loss: 0.003111, epsilon: 0.050000, episode:  483\n",
      "frames: 939000, reward: 11.000000, loss: 0.004091, epsilon: 0.050000, episode:  484\n",
      "frames: 940000, reward: 11.000000, loss: 0.004547, epsilon: 0.050000, episode:  484\n",
      "frames: 941000, reward: 11.900000, loss: 0.003083, epsilon: 0.050000, episode:  485\n",
      "frames: 942000, reward: 11.900000, loss: 0.005856, epsilon: 0.050000, episode:  485\n",
      "frames: 943000, reward: 11.400000, loss: 0.004003, epsilon: 0.050000, episode:  486\n",
      "frames: 944000, reward: 11.400000, loss: 0.003029, epsilon: 0.050000, episode:  486\n",
      "frames: 945000, reward: 11.700000, loss: 0.003784, epsilon: 0.050000, episode:  487\n",
      "frames: 946000, reward: 11.700000, loss: 0.002920, epsilon: 0.050000, episode:  487\n",
      "frames: 947000, reward: 11.700000, loss: 0.003504, epsilon: 0.050000, episode:  487\n",
      "frames: 948000, reward: 12.300000, loss: 0.007031, epsilon: 0.050000, episode:  488\n",
      "frames: 949000, reward: 12.300000, loss: 0.003354, epsilon: 0.050000, episode:  488\n",
      "frames: 950000, reward: 12.600000, loss: 0.004470, epsilon: 0.050000, episode:  489\n",
      "frames: 951000, reward: 12.600000, loss: 0.002250, epsilon: 0.050000, episode:  489\n",
      "frames: 952000, reward: 11.900000, loss: 0.004681, epsilon: 0.050000, episode:  490\n",
      "frames: 953000, reward: 11.900000, loss: 0.003666, epsilon: 0.050000, episode:  490\n",
      "frames: 954000, reward: 12.100000, loss: 0.001259, epsilon: 0.050000, episode:  491\n",
      "frames: 955000, reward: 12.100000, loss: 0.004227, epsilon: 0.050000, episode:  491\n",
      "frames: 956000, reward: 12.500000, loss: 0.003624, epsilon: 0.050000, episode:  492\n",
      "frames: 957000, reward: 12.500000, loss: 0.003728, epsilon: 0.050000, episode:  492\n",
      "frames: 958000, reward: 12.500000, loss: 0.005509, epsilon: 0.050000, episode:  492\n",
      "frames: 959000, reward: 12.200000, loss: 0.004118, epsilon: 0.050000, episode:  493\n",
      "frames: 960000, reward: 12.200000, loss: 0.002487, epsilon: 0.050000, episode:  493\n",
      "frames: 961000, reward: 12.600000, loss: 0.002182, epsilon: 0.050000, episode:  494\n",
      "frames: 962000, reward: 12.600000, loss: 0.003859, epsilon: 0.050000, episode:  494\n",
      "frames: 963000, reward: 12.800000, loss: 0.003110, epsilon: 0.050000, episode:  495\n",
      "frames: 964000, reward: 12.800000, loss: 0.004225, epsilon: 0.050000, episode:  495\n",
      "frames: 965000, reward: 12.500000, loss: 0.006122, epsilon: 0.050000, episode:  496\n",
      "frames: 966000, reward: 12.500000, loss: 0.004952, epsilon: 0.050000, episode:  496\n",
      "frames: 967000, reward: 12.500000, loss: 0.003943, epsilon: 0.050000, episode:  496\n",
      "frames: 968000, reward: 12.000000, loss: 0.002888, epsilon: 0.050000, episode:  497\n",
      "frames: 969000, reward: 12.000000, loss: 0.014819, epsilon: 0.050000, episode:  497\n",
      "frames: 970000, reward: 12.100000, loss: 0.002666, epsilon: 0.050000, episode:  498\n",
      "frames: 971000, reward: 12.100000, loss: 0.005520, epsilon: 0.050000, episode:  498\n",
      "frames: 972000, reward: 12.200000, loss: 0.002306, epsilon: 0.050000, episode:  499\n",
      "frames: 973000, reward: 12.200000, loss: 0.002123, epsilon: 0.050000, episode:  499\n",
      "frames: 974000, reward: 12.200000, loss: 0.002808, epsilon: 0.050000, episode:  499\n",
      "frames: 975000, reward: 11.400000, loss: 0.002695, epsilon: 0.050000, episode:  500\n",
      "frames: 976000, reward: 11.600000, loss: 0.004889, epsilon: 0.050000, episode:  501\n",
      "frames: 977000, reward: 11.600000, loss: 0.003235, epsilon: 0.050000, episode:  501\n",
      "frames: 978000, reward: 12.300000, loss: 0.003629, epsilon: 0.050000, episode:  502\n",
      "frames: 979000, reward: 12.300000, loss: 0.005214, epsilon: 0.050000, episode:  502\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 980000, reward: 13.300000, loss: 0.002448, epsilon: 0.050000, episode:  503\n",
      "frames: 981000, reward: 13.300000, loss: 0.001983, epsilon: 0.050000, episode:  503\n",
      "frames: 982000, reward: 13.300000, loss: 0.002029, epsilon: 0.050000, episode:  504\n",
      "frames: 983000, reward: 13.300000, loss: 0.009072, epsilon: 0.050000, episode:  504\n",
      "frames: 984000, reward: 13.100000, loss: 0.005023, epsilon: 0.050000, episode:  505\n",
      "frames: 985000, reward: 13.100000, loss: 0.008808, epsilon: 0.050000, episode:  505\n",
      "frames: 986000, reward: 13.100000, loss: 0.004414, epsilon: 0.050000, episode:  505\n",
      "frames: 987000, reward: 13.500000, loss: 0.004472, epsilon: 0.050000, episode:  506\n",
      "frames: 988000, reward: 13.500000, loss: 0.001241, epsilon: 0.050000, episode:  506\n",
      "frames: 989000, reward: 14.300000, loss: 0.001074, epsilon: 0.050000, episode:  507\n",
      "frames: 990000, reward: 14.300000, loss: 0.002920, epsilon: 0.050000, episode:  507\n",
      "frames: 991000, reward: 14.200000, loss: 0.003439, epsilon: 0.050000, episode:  508\n",
      "frames: 992000, reward: 14.200000, loss: 0.004051, epsilon: 0.050000, episode:  508\n",
      "frames: 993000, reward: 14.100000, loss: 0.001900, epsilon: 0.050000, episode:  509\n",
      "frames: 994000, reward: 14.100000, loss: 0.001944, epsilon: 0.050000, episode:  509\n",
      "frames: 995000, reward: 15.400000, loss: 0.000959, epsilon: 0.050000, episode:  510\n",
      "frames: 996000, reward: 15.400000, loss: 0.004811, epsilon: 0.050000, episode:  510\n",
      "frames: 997000, reward: 15.100000, loss: 0.002374, epsilon: 0.050000, episode:  511\n",
      "frames: 998000, reward: 15.100000, loss: 0.002960, epsilon: 0.050000, episode:  511\n",
      "frames: 999000, reward: 14.600000, loss: 0.001999, epsilon: 0.050000, episode:  512\n"
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "    \n",
    "# Training DQN in PongNoFrameskip-v4 \n",
    "env = make_atari('PongNoFrameskip-v4')\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True)\n",
    "\n",
    "gamma = 0.99\n",
    "epsilon_max = 1\n",
    "epsilon_min = 0.05\n",
    "eps_decay = 30000\n",
    "frames = 1000000\n",
    "USE_CUDA = True\n",
    "learning_rate = 1e-4\n",
    "max_buff = 100000\n",
    "update_tar_interval = 1000\n",
    "batch_size = 32\n",
    "n_multi_step = 4\n",
    "print_interval = 1000\n",
    "log_interval = 1000\n",
    "learning_start = 10000\n",
    "win_reward = 18     # Pong-v4\n",
    "win_break = True\n",
    "\n",
    "action_space = env.action_space\n",
    "action_dim = env.action_space.n\n",
    "state_dim = env.observation_space.shape[0]\n",
    "state_channel = env.observation_space.shape[2]\n",
    "agent = Nstep_DQNAgent(in_channels = state_channel, action_space= action_space, USE_CUDA = USE_CUDA, lr = learning_rate, n_multi_step = n_multi_step, gamma = gamma)\n",
    "\n",
    "frame = env.reset()\n",
    "\n",
    "episode_reward = 0\n",
    "all_rewards = []\n",
    "losses = []\n",
    "episode_num = 0\n",
    "is_win = False\n",
    "# tensorboard\n",
    "summary_writer = SummaryWriter(log_dir = \"Nstep_DQN\", comment= \"good_makeatari\")\n",
    "\n",
    "# e-greedy decay\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_min + (epsilon_max - epsilon_min) * math.exp(\n",
    "            -1. * frame_idx / eps_decay)\n",
    "# plt.plot([epsilon_by_frame(i) for i in range(10000)])\n",
    "\n",
    "for i in range(frames):\n",
    "    epsilon = epsilon_by_frame(i)\n",
    "    state_tensor = agent.observe(frame)\n",
    "    action = agent.act(state_tensor, epsilon)\n",
    "    \n",
    "    next_frame, reward, done, _ = env.step(action)\n",
    "    \n",
    "    episode_reward += reward\n",
    "    agent.memory_buffer.push(frame, action, reward, next_frame, done)\n",
    "    frame = next_frame\n",
    "    \n",
    "    loss = 0\n",
    "    if agent.memory_buffer.size() >= learning_start:\n",
    "        loss = agent.learn_from_experience(batch_size)\n",
    "        losses.append(loss)\n",
    "\n",
    "    if i % print_interval == 0:\n",
    "        print(\"frames: %5d, reward: %5f, loss: %4f, epsilon: %5f, episode: %4d\" % (i, np.mean(all_rewards[-10:]), loss, epsilon, episode_num))\n",
    "        summary_writer.add_scalar(\"Temporal Difference Loss\", loss, i)\n",
    "        summary_writer.add_scalar(\"Mean Reward\", np.mean(all_rewards[-10:]), i)\n",
    "        summary_writer.add_scalar(\"Epsilon\", epsilon, i)\n",
    "        \n",
    "    if i % update_tar_interval == 0:\n",
    "        agent.DQN_target.load_state_dict(agent.DQN.state_dict())\n",
    "    \n",
    "    if done:\n",
    "        \n",
    "        frame = env.reset()\n",
    "        \n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        episode_num += 1\n",
    "        avg_reward = float(np.mean(all_rewards[-100:]))\n",
    "\n",
    "summary_writer.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvwAAAFMCAYAAAC3emhnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACJ0ElEQVR4nO2dd5wcdf3/X+/dvZK79ORCQhJIgADSpMTQq4AUFfUrigW7yE9RsSHYFRELXwtf0YiAYgMRUJDeIYEEEiCkQnJp5NLuUq7X3X3//piyM7Mzu7N7u7d7e6+nD8zuzGdmPjOzt/v6vOf1fn9EVUEIIYQQQgipTCKl7gAhhBBCCCGkeFDwE0IIIYQQUsFQ8BNCCCGEEFLBUPATQgghhBBSwVDwE0IIIYQQUsFQ8BNCCCGEEFLBUPBXGCJyiIi8KiIdIvKlUveHlA4ROUNEmkrdD0IIGW6IyCYRObvU/SCkUFDwVx5XAXhGVceo6o2l7owXEXmXiKwUkU4ReUFEDnOsqxGRX4nINhHZKyK/E5Eqx/q3iMhTItImIo0i8l7Pvj9jLu8UkUdEZF/HuvEicruINJv//cCz7Uki8pI5UFouIqcU8TKUJSJyrYisEJG49/p42v1JRFREDsqyvy+LyEYR6RKRNSJycME7TQghhJCsUPBXHvsDWBW0UkSiQ9gX77HnAPg7gMsBjAfwXwD3i0jMbHI1gLkAjgBwMIBjAXzH3DYG4D4ADwCYCOAyAH+zRKSInA7gJwAuMtdvBHCH4/C/AlAHYBaAeQAuFZFPmttOBHA/gF+Y/fo5gP+KyIQ8zlFEZMj/rhzXcDA0whgwPpjhOKcAODBEfz4D4NMALgQwGsA7AewqQB8JIYQQkiMU/BWEiDwF4EwAvzWj3AeLyJ9F5Pci8pCIdAE4U0QuNG0/7SKyxRnNFZFZZvT2k+a6vSJyuYi8zYx8t4rIbz3H/ZQZwd0rIo+KyP4BXXwHgAWqulBV4wB+BmA6gNPN9e8CcKOq7lHVFgA3AviUue5QAPsC+JWqJlT1KQDPA7jUse2/VHWVqvYDuBbAaSJyoGP9z1W1W1U3AbjVse+TAOxU1X+Z+/4bgBYA7wt53Z8RketE5HkA3QAOEJFDReRxEdkjIm+IyAfMtrPNaxgx398iIs2Off1NRK40X3/SvK4dIrJBRD7naHeGiDSJyDdFZAeAP4nIKPN+7xWR1QDeFqb/Fqp6u6o+DKAj4DxjAP4PwBVZrkcEwPcBfEVVV6vBelXdk0t/CCGk1JhPnn9tPnneZr6uMddNFpEHzO/0PSKywPHd/k0R2Wp+f78hIm8v7ZmQkQ4FfwWhqmcBWADgClUdraprzVUfBnAdgDEAFgLoAvAxGNHsCwH8PxF5j2d3xwOYA+CDAH4N4NsAzgZwOIAPmBF1mNt9C4Y4bjCPfwf8EfM/7/sjMqyfISLjPMud6zNtC8d6+KwP2ta7PgyXwnjqMAbGYOFxAP8AMAXAhwD8TkQOV9WNANoBHGNudyqAThF5i/n+NADPmq+bYUTGxwL4JIBficixjmNOhfE0Y3/z2N+HEX0/EMbg6uOuEzIsUr/L4Zy8fAXAc6q6PEu7GeZ/R5iDxo0i8sNSPPkghJBB8m0AJwA4GsBbYTwh/o657msAmmD89u0D47dQReQQGIGRt6nqGBjfx5uGtNeEeOAP8MjgPlV9XlWTqtqrqs+o6grz/XIYAv10zzbXmm0fgzFAuENVm1V1KwxRbwnWzwG4XlXXmFH7nwA4OiDK/ziA083odDWML8dqGFYbAHgYwJdFpEFEpgKwko7rALwOQwB/Q0SqRORcs8/Wtg/BGIgcJSKjAHwPgDrWPwLgahEZY3rPP+VY9wKAfUXkQ+a+Pw5DNFvrw/Bn8+lCHMB5ADap6p9UNa6qrwC4B8D7zbbPmtdhqvn+bvP9bBji/jUAUNUHzci4quqzAB6DMUCwSAL4vqr2qWoPgA8AuM58QrIFxhMSG1X9vKp+PodzshGRmTDu9fdCNJ9h/nsugCNhPHX6EAyLDyGEDCc+AuBH5u9fC4AfIvVkeQDANAD7q+qAqi5QVQWQAFAD4DARqVLVTaq6viS9J8SEgn9ksMX5RkSOF5GnRaRFRNpgeOone7bZ6Xjd4/N+tPl6fwC/MR9ptgLYAyM6Pt3bCVV9HUbU+bcAtpvHXA0jQgIYTyFeBbAMhgj/D4wv1GZVHQDwHhhPJHbAiKzcZW2rqk/CiHDfA2AzjGhKh2PfXzL7vQ5GLsAdjm13w/D+f9U8z/MAPOHYNgzOa7w/gOOta2Jel4/AiMgDhuA/A0Y0/zkAz8AYvJwOw/KUBAAROV9EFpuPilsBXAD3fWpR1V7H+309/dicQ/+z8WsYP3ptIdr2mP/+XFVbTQvVH2D0nxBChhP7wv1dutlcBhh5X40AHjNtl1cDgKo2ArgSwA8ANIvIneIoIkFIKaDgHxmo5/0/YCSpzlTVcQDmw98yE4YtAD6nquMd/41S1Rd8O6J6t6oeoaqTYAj0/QEsMdf1qOoVqjpdVQ8AsBvAy6qaMNcvV9XTVXWSqr4DwAEAXnLs+yZVnaOqU2AI/xiAlea6Par6EVWdqqqHw/jsO7d9VlXfpqoTYURvDnGuD4HzGm8B8KznmoxW1f9nrn8WRqT+DPP1QgAnwxD8zwKGb9Q8hxsA7KOq42E8xXDeJ+993Q5gpuP9fjn0PxtvB/ALEdlh5gwAwCIR+bBP2zcA9Pv0jxBChhvbYPxOWexnLoOqdqjq18zfq3cB+Krl1VfVf6jqKea2CiNnjZCSQcE/MhkDYI+q9orIPBge/3yZD+AaETkcAERknIhcHNRYRI4TkaiINMCI+v7XjPxDRKaLyL5icAKA78IYFFjbHiUitSJSJyJfh/Eo9c/muloROcLcdj8ANwP4jaruNdcfKCKTzGOfD8Pz/mPHvo8x7TxjYYjsJlV9NM9r8gCAg0XkUnOfVWIkPb8FAFR1HYwo+EdheOLbYTxZ+B+k/PvVMB4JtwCIm30+N8tx74JxLyaIyAwAX8yl02Y/a2F8L8TMa2pVdToYhn/1aPM/wPiB+7d3P6raDeCfAK4yLVQzAHzWvC6EEDKcuAPAd0yr6WQYtsa/AYCIvFNEDhIRgZGblQCQEGM+nLPMwE0vjO/7RIn6TwgACv6RyucB/EhEOmB8ed2V745U9d8wIhd3ikg7jIj6+Rk2+Q2AVhhR4FYYQtDiQBhWni4AtwO42swhsLgURhS7GUbE+RxV7TPX1cJ4ctEJIzK/CMaAweI4ACtg2HyuB/ARVXWWL70KRtnILTAGEnaNfxE5VUQ6M5yTC1XtgCHOL4ERCdoB4xrVOJo9C2C3qr7peC8wLE3WPr4E497shTEouz/LoX8I43HzRhh+/786V4rIfBGZn2H7P8L4YfoQjES1HpheVdO/usP6z2y/y8wd8Nv3FTDuxTYY9+IfAG7L0n9CCCk3fgxgKYDlMH5DXkEqWDQHhv2zE8b33O9U9RkY3/U/hfGbsgNG8YZvDWmvCfEgRn4JIYQQQgghpBJhhJ8QQgghhJAKhoKfEEIIIYSQCoaCnxBCCCGEkAqGgp8QQgghhJAKhoKfEEIIIYSQCiZW6g44mTx5ss6aNavU3SCEkLLk5Zdf3qWqDaXuhxMROQ9Gud0ogFtU9aee9YcC+BOAYwF8W1VvcKwbD+AWAEfAmJzoU6q6KNPx+DtBCCH+ZPqNKCvBP2vWLCxdurTU3SCEkLJERDaXug9OzInZbgJwDoAmAEtE5H5VXe1otgfGnBLv8dnFbwA8oqrvF5FqAHXZjsnfCUII8SfTbwQtPYQQQvJlHoBGVd2gqv0A7gRwkbOBOWnbEgADzuXmrNanAbjVbNevqq1D0mtCCBlhUPATQgjJl+kwZqe2aDKXheEAAC0A/iQir4rILSJSX+gOEkIIoeAnhBCSP+KzLOz07TEYvv7fq+oxALoAXO17EJHLRGSpiCxtaWnJr6eEEDKCoeAnhBCSL00AZjrezwCwLYdtm1T1RfP93TAGAGmo6s2qOldV5zY0lFXOMiGEDAso+AkhhOTLEgBzRGS2mXR7CYD7w2yoqjsAbBGRQ8xFbwewOsMmhBBC8qSsqvQQQggZPqhqXESuAPAojLKct6nqKhG53Fw/X0SmAlgKYCyApIhcCeAwVW0H8EUAfzcHCxsAfLIU50EIIZUOBT8hhJC8UdWHADzkWTbf8XoHDKuP37bLAMwtZv8IIYQUwNIjIjNF5GkRWSMiq0Tky+byiSLyuIisM/+dMPjuEkIIIYQQQnKhEB7+OICvqepbAJwA4AsichiMagtPquocAE8ioPoCIYQQQgghpHgMWvCr6nZVfcV83QFgDYw6zBcBuN1sdjv8Z1kkJC/2dPVjeVNrqbths2pbG3Z19pW6G3nT2RfH0k17St0NX+KJJJ5v3FXQfSaSioXrCrtPMrxY3tSKvV39pe4GIYQMCQWt0iMiswAcA+BFAPuo6nbAGBQAmBKwDesrk5y5ZcEGXHrrS6Xuhs2FNy7EOb98ttTdyJtv/Os1vH/+orIctNz45Dp85JYXsXjD7oLtc/6z6/HRW1/Es2v5nTNSefdvn8f7579Q6m4QQsiQUDDBLyKjAdwD4Eqz+kIoWF+Z5MPe7gF09sVL3Q0Xe7sHSt2FvNm0uxsAsKOtt8Q9SWf9ri4AQHNH4QYj61s6AQC7CrhPMvxY39JV6i4QQsiQUBDBLyJVMMT+31X1XnPxThGZZq6fBqC5EMciBAC6+uJIJBWJZNhJPUkmJo+uBgC0lGGE328q18Gi5scmwplICCGEjAAKUaVHANwKYI2q/tKx6n4AHzdffxzAfYM9FiEW3f1GdH8gkSxxTyqDhtE1AMo74q1auMFd0tyXFGU4QQghhJQXhajDfzKASwGsEJFl5rJvAfgpgLtE5NMA3gRwcQGORQgA2Hae/kQStVXREvdm+DN5jCH4yzLCL4UX5daToSLsmhBCCCk7ClGlZ6GqiqoepapHm/89pKq7VfXtqjrH/Lc8S4CQkpBMKl7aaHwkVm9rx/KmVmxv6wm9fXd/AgAwEGeE30tjcwdacozU11Ubg6bdne6qJbs6+9DY3AFVxYsBSbPWOm8EfkVTW2CexUsb92Dhul1Y7LNd70ACr765135fTEtPNFIcxb9meztau41ruaKpDV1llm9CCCFkZEEHKykJNy/YgA/8YREWrGvBBTcuwLt/+zxOvP6p0NtbAmogQQ+/l7N/+RxO/8XTOW1jCWBvlZ4zfvEMzv7lc/jbi2/igzcvxiMrd6Rte8dLW/DBmxfjoRWpdf3xJN7124X43F+XprWPJ5L4wB8W4aO3vohLbl6Mf73c5Fr/zXuW472/ewHN7e4E4gI6emxLT6RIIf7zf7MA7/v9C+jpT+Bdv12I//f3V4pyHEIIISQMFPykJKzd2QEA2N6aX1WYrj4zwk8Pvy/WE5CwWFH25na34Lci9Oubjao2W1vTn8Js2m1UOtmyt9te1hc3jr9k09609n2epzKvbWl1vV9iPvnpN+9tMTR5SvAXft/WtdzQ0oWBpHEOr25Ovw6EEELIUEHBT0qDGa21BFGudPWnPPxk8FjFjpo7/Adgaie5piN2m9QyS9T7tfcK/j2eyY96zfUxTwkdReFC/NbHphj5AXFH5ahCPpUghBBC8oWCn5SUeB6WHFW1LT39ZebhL7f+hMWKeAfVurfukm9EXKw2qXvZO5BwbefEWmfhtRH1mE8nEhkGGYPFGsBEiyD4neeXNMU/dT8hhJBSQsFPSko+lpy+eNKOSJeDpceZdGqVCx1uWNezozeeJsiBVKQ6bETcG8XPtM6bKNxjHj/pmWOhGB7+YuA8P2vQUsiSooQQQkiuUPCT0LR299szlObDlj3daO7oRXd/HGt2GB5+v6TbeCKJx1fvxKZd/rNgOiueFEPwb23tSZtxtqN3wM478OKc/GuoZv/duKsrzQozGJyC1K/Cj2VT8YvwW7XsXZaegUyCP3OE38Kule8ZZGzZ040129vx5u5uv81c7GjrxdbWHqzd2YGO3tRMyNYtS4QQ4k17u9MSiDPhEvyM8BNCCCkDClGHn4wQLrxxIba29mDTTy/Ma/tTf25UjjnzkAas2d4OwBD3XhY27sJn/7IUBzTU46mvnZG23krYBYD+eOGl1Mk/NaoFOc/zU39egiWb9vqeuzMQ3ZNjsmy+nHnDM5hYX41XvntOQfbnjHjv7urHzIl1rvW2VSl0hN+4Dr4efs9goL3Xf5DknUXZ6qL1OQKQ9bN4wvVP2q+PmD4WD3zxVACp8w0TeT/lZ0+HOpZFn+MJSZwzQRNCCCkDGOEnofGr0JIPizekpmTwi9DvNeuXN+31P16XwzYzVEm7VrUZrwgF3GJ5KPVdISP8zn77naMl4H0j/D7LrCi33+XIZPdx96mwHv6VW9vT9l2Mj4/z/Ly2JEIIIaQUUPCTkjLgI4isCH5VQM1El6VniJNk/ZJynUHiQlaSGUpc5+AT9U5V3Um/J353yS8PIMw6J96Pht+VzddCZZ1iGEtPrvT6RPhp4SeEEFJKKPhJSfFaelTVTnwNmgW1y2GbGeqkXb8nCq4If+lziPPCeQ5+2tQa6GRy9DgHCrkk7QZhP2mwqgD5qOZM3vpMAwvrfIsRgXd7+IfpB4IQQkhFQcFPSoo3Yj6QUHSaEf4gwd/dN/SWHkvo+g0w3Jae4RnKVdegpXCWHv86/OEi/Jbgt5OCfdr4JRhbeCcRc2In7RZZ8Fuf5eH65IcQQkhlQMFPMtLTn0Bjc/6VebLhnRG2L56wBX1/PIl4IonV29pdbTpdVXrcQmpney+een0nevoTdmIwAGxo6cQjK3dgy55UZZdV29rSosabd6dXBtrd2WdbMpY3tdrLVRUrt7a5rCertrUhmVR7nUVze2+oSi+bd3dhwboWX9G9va0HuwMq2mRjT1c/nlyzE43NHWhs7kyrQuQ8nJ8G7s9g6bFQBd7Y0YEXN+zGK+bMst5dNbf3+uZmvLm7G+29A2jrdlbSUazd2YG4GSXv6I2n3Z/mjj40d7iv7d6ufqOyjs8kYq++uReqal/fpCriiaTrszJYnEm77T3G+QzTcSAhhJAKgVV6SEY++5elWNi4Cxt+ckHB9umMdvYMeAV/0k7K7YsnccNjazH/2fV48mun48CG0QDcgwTvE4Jv3L0cz61twfTxo7C1tQePfeU0HLzPGFz215fR2NyJY/cbj3s/fzIWb9iNS25ejO+/6zB88uTZ9van/+KZtP4e9+Mn7Nef+vNSPPDFU3DE9HG4b9k2XPnPZbj+fUfa6795zwo0t/dh1uR6fPGOV/HbDx+Ddx61L+b9xKgWk63Sywf/sBg72ntx9+UnYu6sia51J16fXj0oLNc9uAb3vNLkWubcj8vSk8HD7/cEwxmBf8evn8vYD+s6eDntF0/jgIZ6nHXIFHvZ9rZefO6vL9vvr31gNa59YLVru+aOPsy7zn1t3/7LZ7Gnqx9/uPS4tOO893cv4IaL35qy9Kjixw+uwZ9f2IQFV52ZVp0oH3odn8l2sxQo9T4hhJBSwgg/ycjCxl0A3DaFwU4i5BTpXp9170DCTtqNJxWL1hvHd9ozOjPU4d9rVq6xKgpZlg9rubWtFWVe4YjChz2vnWY0+Q2zLr/3CciCdbvsmv25Ph3ZYe67K0N5z0y+86BzsCofBe4zS4Tfuv9+Sa5ie+wzHiIrG1q6XOe9N0QVIr8ovlW9KChXYF1zh8PSAzz9RnPG9rnijvAPz4nYCCGEVBYU/CQUTt/1YIWdq269T4TfOVutJcJaHYLVud4r+Ktj/h/pTodNCACiZjOnh7s3w2RRTqzcgqDr0OKw3WSywGQik6jPVNs9aFU237xmyUOwrpuf5z3TGeZ69s7zDlNBpyWDTz9owBAVsc83oepoV5g4fJ9PhL+SQ/wicp6IvCEijSJytc/6Q0VkkYj0icjXfdZHReRVEXlgaHpMCCEjDwp+EgpX5ZECGpK9E1X1DSRdEXzruE4R3dWXsIW9N2nXT2DGE0l7P5bnPxoxtneKZ2d9/0ykBL9/ycVdHSnPf0DecVYyJf9mSjQN2s45WVm27fz2YQ0YMh3bLzE11yRmd8Wj7Ns2Z0jaDZo3IhoR+zOsqvbEXwWL8DsFf89AhpbDHxGJArgJwPkADgPwIRE5zNNsD4AvAbghYDdfBrCmaJ0khBBCwU/C4ZwdtZCVTXo8UfXeeMLl0bdq7jstPV19cUyoqwLgU+XHr66/0/NvDhBiphJPOJJ+u0LWdI+aHpYgMdvRF7fFb8iJadPIdI3jGUo9BvWpO8tgxnk4axdOwZ0pwm+dpN+6gYTmVPoy4Rp4ZG/vV6WnKmr0J0jwi4hdPtXZZ28CeL44bWp2hL9ymQegUVU3qGo/gDsBXORsoKrNqroEQNrFEJEZAC4EcMtQdJYQQkYqFPzERWNzB/rjSWxo6XTbXRyWkCBR2R9PYuG6Xdjd2YfegQQ27+7Cnq5+vLm7O1B89XiEaN9A0iW8rQjuiq1tWLezAwvWtWB7Wy/G1FYhFhHb0rNpVxf64om0uv5tPQN4dOUOAEBNLIKBRBJNe7tTEWvHuSxvakMoxLoOwU2s3YpH8a/d2YEVnuPs7uzD5t1drmvUY16/dWYugLO6kPO+dPXFXev8bk1zh39lHCd+EX7n05O+TILfJEgw9yeSWN/SmXZv/HD2P8zA0s/DP6GuGgCwNeCcE8kkNu7qSjvGQMJYbn02Gps7ffuwt6vf97jWOue1tjz8VsWhCmQ6gC2O903msrD8GsBVADhhASGEFBFW6SE2y7a04j03PW+/f/TK0+zXzgh/kA77z6tbcdU9y3H2W/ZBXXUU97+2Lesx/cpydvUlMKYmhg6H8H92bQueXdtivz9qxjhURSPoNz3/Z9zwDN5z9L6Ie0TnDY+9gQ0thrgbX1eFzt44TvnZ0/Z6S9DtaOvFF+94NWt/AdjHyORWCVp17q+MKjavfPccTKw3hKmzCpDF3xe/iS/fuQwA8Ow3znBVD3LakN4/fxHWbG+3K9T4CVSrik1GfJJ2B3wEv1/+gDWkCZoEbUNLFy64cQE+cdKsrN1w9j+MHWhvd3oEfXxdFZo7+rC9zV/w3/T0et9j7GzvxcXzF+HDx++HT508G2f/8ll86ayD0rY/5trHAfhXS7LWWVgR/nhSce6vnsPfPn08TpkzOet5DSP8nmGFelQiIu8E0KyqL4vIGVnaXgbgMgDYb7/9cuwiIYQQRviJjbfG+R5H0qMz6TMo8rplrxFp3tneixfW7w51TG/Sbjyh6OqPY4IphgHg06fMxlmHTnG1q62KoqbKEPxWsu1z63ZhwGN36exNDRom1FWnef4tARt2MihjG3eJSj/veirC77+PbQFPPCx2OXIWWj2i1jmo8daPz3fiL7+ynM6IvXXPMyYTe67tde89AgDsaPqCdS1p22TqR6bk5EzEzPyMbHkLxvFSrzfvNj6/r77Zal//sJ9jL1buhtfDvy1gEDKMaQIw0/F+BoDsI32DkwG8W0Q2wbACnSUif/NrqKo3q+pcVZ3b0NAwmP4SQsiIhIKf2Hi1orNyizMRMUj0WX7q3Z19GDsq3MMjb1nOeDKJ7v6ES/DvO34UJo+udrWriUVQE4ugdyBp+8tVNS3C7xSN4+uq0mwniUx++IDzDOP1tj38AXVqgiwhFs7rkj5IyeThz9q1rNv5RfhTxw4uy9nvuS4HmfMmtPYYA8cwVZBcgj/PWZRNve9K/g7COXi1nghMqq+2k8IzlUfNRE0silhE7IRgi7G1VXntr4xZAmCOiMwWkWoAlwC4P8yGqnqNqs5Q1Vnmdk+p6keL11VCCBm50NJDbLzRYae2cwrQoCo9lt++pbMPh+87LtQxveK5sy+BRFIxySH466ujdmUci5pYFDWxKPriCVd03isSnf0eP8o9aABSgs9PyA4kk6iJRNOWW4MKO8LvdzmyCG+/ZFMnzicf3kFRxko5BYzwexOive28eK/9lLG1AGDP6humCo5zLOMdQHipjkV8+5gLzmtp9XPS6GpUm3Vbvdc+LNGIoDoW86nSU1n1OVU1LiJXAHgUQBTAbaq6SkQuN9fPF5GpAJYCGAsgKSJXAjhMVQs3vTEhhJCMUPATG6+OTAZF+AMFvyGYBhKat/C06u1PdAr+mlha8mtNVQS1VRH0xZOuvnmr9LgEf116dNUSfH4iOp5Q1Pj8hVgR9jAefhF/Ed6coX484L7e3tKlmawu+VZQcnYx5wi/+RTD237KmBoAwLZW43MRRjwncojw11VHfQV/Lg8GnMfYbgr+ifXVqDIFv/fah0XEGKh6cwwKVfqznFDVhwA85Fk23/F6BwyrT6Z9PAPgmSJ0jxBCCGjpIQ68wtQt+B1VepKpqjjbWnuwu7MPzzfuwvbWXtRXGxHxfK0QVkTUFeGvidqlMC1q7Qh/Eq/vSFU/8YpEpz4dX5ce4X9zTze2tfb4TtLktQdZDHgi/H5Y11LgL8KtpyFBAyOnOO71iMTMdfjd74MSV4FURLuzL47mjl7bd+5Xpcfev8+xLfuS92lNfU0M9dVRuw9hLDaqapfVDEoCtqiNpT99AXKzAq1vSeWtWFWS6qpT+81UztR7/ZxERFBfE0ufWC7k5G6keCSS7oDEkk17StgbQggZGhjhJzZe7ekUvE6h8vjqHfjufasQEUNgnnFIA555w0jIPO3gBjy3tgXbsySlBtFqCf7RTktPLG0Cq5oqw8O/Ymsbnnq92bfPXibWp0f4d7b34aSfPuXbPsgrbwnKTMF0KxIu4u/5391lCP6giK9z330+ic1BeAcQJ17vf24AcML1T2LTTy/Eeb9+Dk17e1AdjaA/kUwJfp+++UX4rUGA81ys8dmk0TXY2Z45X8FJIqmoikYwkEhkzZWoqfKPV+TylOPBFdvt11ZydCKZGvQ48w5U1fWkybp+773peaxr7sSGn1xgr4uIe+Bg0ZtDcjgpPHu6+nHstY/ju+9MzQ128fxFvhWXCCGkkmCEn9h4I9bOCKsz4rxia5vZ3ni/vbUXx+w3HndffiK+fcFbAOQW4d9/Uh0e+OIpAIy6+QCw38Q6e319TQyRNA9/BLVV0TQvvLdKDwDM3X8CFl/zdt8IfyaCrDOWbSg1066/HQgw7C5+kXJLlIaZ7CvfCL/z/n1o3n6+liZVtevGp2YQhtm39Hvod2xrkfUU6BfvPwqvff9cAMYkWH7JulYU329f1qRoftfNieWz9xKmuo9fuU2LRDLp+5QjaLfrmjsBAN2Ov5GICGqq0gU/I/ylxaqOdc/LTSXuCSGEDC0U/MTGK2j8Jl4C0p8E7O3ux77jRmHurImYMWFUzsfdZ2wtZk4wBL4l+BtM/zdgCn6vhz8WRU0s/ePrF/2urYpi6rjaQIEYRJClxIrwW9fBTwdaTwdEgpJfjX/DlI5Mi/D7DGqsQYfTA9/t2PeBDfW++3ZG0S2hbYldPzuLn+C3jml9RiaNrrar0UQj/gOeqoB7kVRFzFw3kMXvHrSPMBH+yY7PV/r2/vkZftfdeSznRF8iglo/wc8IPyGEkBJAwU9s0iP8QUm77u1auwds+4Ll286FKWNqEDMjvpatoq465Tbzq9JTWxVJs3QojOiuV9hb+w4SiEEEWWe8VXr8sK5dIqm+Awc7ip7BI24RJmnX2p/TY+/ct1+0GXAL0Igt+I33fp57X0uPR/BbdfABI9LtN+CJeT1ajn2F9fBXeQZ81rlnKltq4U0Cd+KM8LuX+1RySiRt+5I13wBgWHr8BqSVmLRLCCGk/KHgJzZea4p7ptVEYLv+RBL1jnI2VjnGsEwZU2uLcitpd7Rjf0aVHvc2VllOJ5YgG+UZcFgCNMhG4m6bahMkHC3bkHUVfKPBidTMtP7CNbylxysS/QYifn1x7ttPfHr3nR7hT49G+yXt2h5+80lEzHGdoxHxjWpXB/QnmUzdr2we/mrP/bSeNISJ8HuTwL378duF32CnP5HEpHrjacGm3U7BL77XPN8yn4QQQshgoOAnNl494/bwZy7LWV+TEtkNo4PtEn40jKlBlSnyLEuPM+FxVFV6lR7Dw+/++FpC2PuEwRL6QSLTiVOsDiTUt3pPImF594P3Y/n8kwGC37b0hMh1CFOH3/ImOy09zn2HEfzWU5S+eBItHX2+gxH/CL/xrxXJdz5JiUX8I/yBdhxHhD+bh9+7j0xzKnjJ9LAnkVT/CL/PAOTN3d12RalNngi/n6Xn6TdaXLMoE0IIIUMBBT+xyWTpcUZ7/fSUM8LfMDY3wT9lTA0iEYEIsMesw19fE8O8WRMBGFYTr4e/KippEX4rIp8W4TfVXRgPf5XDjhJPKI659vG0Nt6kXT8s/3k8qeiP+1lwzCh6qKTd7B7+U3/+NLbs6Xbdw25XhN/f0uMcTFgR/u/+ZyXedt0Tdn6Bc/CVyeqSsvSk7lUkIr6flyDBP5BI2vcrW3lN7wDOGlhlqmJk9yujpUd9B3N+CeHv/L+F9iD1zT3d9nIJiPA3NnfisVU7s/aPDC3W/B+EEFKpUPATm6AIvwiw2xGV9I3wOzz3UzIkRFp8/owDU+3NAUJVJAJVYExNDLVVUdz+qXlYdM1ZAJBWpQc+gsoaoDj9/8Z+TQ9/iAi/s41X4M2caCQkp8pyBgtLS/wmkuobqba2DFOb3vl0ZWJ9daBlpWlvj2sQ4ty392mI3U/Hvr3XeHtbj5Er4bgm/vkDluA3BghOMR9knQmyV/UOJO3ts1l6giP82X3ymQR/POk/cVzQdd9hlh3t6E1db5HgQVaYvw9SePriCbzz/xb6rjv6R49jpVl9jBBCKhEKfmLj9WdbUep9x42yJ4oC/KvSOKPAU8Zk9/BPHZdqY7W37DTWE4JR1VFMG2eIbL8cz6BE1PQIf/ikXacQ9UaKp4ypxaiqqC16rculPlfEEr8JzWzp8fPJe7Gi8GNqYkgkNdCyEk8mXYO2bpelJ3vSrjeRduOuLtRXx+yIO+Dv4ber9JiDB6ctKm2gZhJ0L/oGEqEtPd4nNtZ1CeXhD+gXYJyjr4c/ywDEmSQdEQkcZE3J8QkYKQytjlmPV29vT1u/rrkjbRkhhFQKFPzExiskLaE6ffwoV717v+inM8k2TATTGWG12luC02/7tEixaqAv3evhtwRruKRdR4TfIzhVFbGo2MszyT8rKp9Iqm95Sesa5hLhr4pFkEhqoKAdSCRd69xVevyvlfPpgVcEb9zVhbqaaNZE5lQd/vQqPUER/qB8it6BhH28rFV6vEm7Ph7+oOMEDUSs7YOsS5lsXM4yqJGMEf7cktoJIYSQwULBT2y8nmnLq77v+FqX4PdzTNS5qvRkF/xOcWlNCGVFff0EkZ9ACxL8QZaeoPautg4R2dE74FqnZh+9ZTn9NGBWS48d4Q9TpccQktXRSMYIf3/cXU7SmXQbJLydEX6v4G/u6EN9dcy13E+DW1F/6zyrPFV6/AiM8MedHv7cLD3xpCLp8d/7zXYLZK7Sk1R/wW9UXAruk/N6GxNv+Z+jcxZpQgghZCig4Cc2A0GWnvGjXKLVr8rIaGeVnhARfqcQtGqiW1YQvwh/muc6YGIjIDhpN4ylx2lfcVoAAEOkxyKCrv442roHMob4rRKViQCRqFDs6uxDZ2+YCL/pjY8JEqqByazbWnuxqzOVfOic1CsREJluckwW5edrr6+JuSL8ffGEneBoDQK94th5DYMi6UF1+Hd39dtWnWwRfm/0PpFIHwyNCviMZAjwI57wT9o1BlvBfers93r4/T9vuc4HQQghhAwW/vIQm7QIv/l+2jh3xH3p5r1p2zqj6tPGjcpaEccvwmqVb/QbMHgF2gGT6zGpvtpXuHmjutZAwmk7CuL42RPt11b1FQsrwn/vK1vx1h89lorw++yn1yX400ViW88A5v74Cdy+aHPWPtmWnkgEyQwR/h89sBofv+0l+73z6cHEumqcNqchbZvv/GdlxmPXeSY9W7BuF47+0eNY0dSGt133BO5+uSkt6l/laB/konKK3lPnTHati4X08B85fZzrfTyZTLM7eQd/FpksPcER/mTGCL9zk0iGASkhhBAy1FDwE5s0D7/5fuyoqqzbOqv0jBtVhUeuPBVnHTolsH00Inj5O2djybfPtpdZwnZMbfrxLNE5eXQ1HvvKaTj5oMk49/CpeOwrp2H5D87Fp0+Zbbf1ijyr1Oak0TW4/4qT8f13HWav++8Vp+COz55gv//y2+fgtx8+BoC76oqFMyE1k6Wn11GW00/wO73zBzbU4/GvnIZ1152PJ756elrk2o7wRyNIaLCH30t/PIkxNTE88dXTMWtyPX5x8VE4dOqYUNta1Falz3IMAOtbOgEAz61tSfO1OyP8QZYe5/I/fmwunvra6fZAzRoM+NXvt7j2PUfgPcdMdy2LJzWtstLUgEngMll64hmSdq1B8eH7jg3cHkifeOvpr5+BV797Dl761tszbkcIIYQUAwp+YuMVppalp746e2TcOfEWABzQMDqwSglgRFgnja5xRfOtiK53X0DK9gMAB+9jiNZoRHDQlDEYW1uFfcePSvXF01+nSD9qxngcOjUl1mZNrsMhDhFcHYvgEHP/7R4PP1RdVhRLePslclqWnmRAkq0zglxfE8OcfcagKhrBQVNGp4lRqw5/VUygmr0+vUV/QlFTFcVBU0YDMJJID2ioD7WtRU0sAkFw/kRfPJFmF/LOtOuHc3ltVRQHNIzGcftPAJDKAchk6XnL1DHpHv6Epk2OZVV5ynR8L0ETb8UdT1e8Txe8eMtyzp5cjwn11TnPQk0IIYQUAgp+YuNNkrQE1+jaMII/vY2fULTwi7BawtibdGu0z3x853qvpccrDJ1iLyKSlmRq2T28EX7L0mNheeT9BL0zwu87O61jmXeA4hWjtqUnaonscIJ/IJFM85FLhsi237raqqhv2VErIbUvnkyLhjsnLwsS1n75AtZgKhrJnrQbjUjavv0Smr12NAvv4Z3XKagaj9OeFWQVsshUlpMQQggZaviLRGy8CYkDCYVIcKUTi2jEf1bRDNoSmSz+fhH+TJ5rqw8WaUm7nm2dbaMRcYn4WCRiD0bSqvSoO3ptldT0E/zWsoSPrxxw26e85+s9VetpgSWks3nbLQYSybTSlZkmnPJbUxOL+FqW7MHHQDKtNr+rDn/A8fxupyX0o2Ksz3SefgMJPw//tPEBlh7P9s7PTDxgpt14MmkPQrL9TUQiwWU5SWnIVpQ3U4CCEEKGOxT8xMabkNifSKIqEskqXOqqo77R4UziMtM6PwtRpvaAe0CQnrQbHOEX8cwM64gct/d4I/zqqjFvC/4MtdkTmp4bIeKJ8HuejngHN9bTgqqYmcwaMsLfH0+mPd3ING7yu8Q1sahvUrIlfHvjifQqPZ4BVdhjWdtFxLgHmSw9fvtt7R5wlRkFjEnjfLf3dMA5YA2aeCuRVDuR2+8plJNMZTkJIYSQoYa/SMTGr0pPLOofvXcS5PE/akawzzmTh9o3wp9N8DvWj6ry1OH3RLldglTc1pCYQ/C39vS7tjtuvwmu6kOW5SdTEm0imUTCc11jEXENAibUueuye8WoJfAt8d4zkH12XsCK8HsFfyZLT/qy2qqIr73Fir73Dbij6rGIuAZ/mZJjvUTN+xSJCCIiGSviOAdeFp/88xJcdfdy17L9J9X5bu8dVDkr6gRNvBVPKC666XkA2SP8AiN5HQie/IsQQggZKrKbs8mIIa0OfyKJWCR7eUE/gQ4Anz5lNpZs2oNHV+1MW5fJouOXD5CtdLlTWKZbeoJFr1cARxyCf2d7ar6B0w5uwLcvPAyf/ctSe1lnnxHt9ROHFn6+8mhE7G0uOHIqrjx7Tlof/LAGGy3tfRhTG/OtIuSkP6FpYjOT/vZPzvWP8FvRdyPC72zvudZBEX6fY6Ui/MZ9SWSoeW99Hp77xplYu7MDnzHvy4sb9wAAfvY/R+K4/SfggIbRuOtzJ2LcqCqMHRXDidc/ZR7DI/gdT7GS6u/hd1qMsgp+EcyZMhr/+OzxnFmXEEJIyaHgJzbpEX5FVTSSPcIfUN9eRHDk9HG+gj9o4iXA3y6RKdkUcAvLWk9/Y94Iv9Nj7tMPSww66/C/ZeoYVMcirsGNlUybQZf6Cv5YJGJHxc85bB+MzxLht7Ci9dvbejFlTE1WwT8QT6bNh5BLxB1AoC3FeurQN+Ce3bfGMzjMKcJvJ+2mJ+SmY6zfb1IdWnwngqvCQVOMakvzHHMrpI7lfu9MsI0n/C09zgnnRmW19Bif2ZMOnJyxHSGEEDIU8FkzsfGr0hOLZvciZyrbGSTUMwnBep/oaTbh6BRw3icSuSSuGvvyGQSYy/wGIxk9/D5lOWPRlKXHry9BYtc6jx3tvaGixv2JpO37t8h27l5qYhHfmcVswe/x8IeN8PthJSWLSMZcA4PUMatcScLGv9kGDN7r4ByoBE285ZyVuC7LU69sA1RCCCFkKKHgJzb+lp7sSbtBlh4gQ1nGDILMm2RrtM/YBZeA8wp+r6Un09MFwH9wYW3iN1uvt0qNk3hS0wZSURF7Gz9hGKQVUxH+Ht/ZiL34evgzXEeFpgltoyynua1j3UCAh9977bNZsVxtLQ+/ZK/f77zkzrbWJHHeQV7QfizSPfzp22xrTQn+bL78kST3ReQ8EXlDRBpF5Gqf9YeKyCIR6RORrzuWzxSRp0VkjYisEpEvD23PCSFk5EDBT2zSLD1xRVVUzNKVuVlwLIK0dXbLhnc/4SO23n1nKsvpuy+fvwpr/37e7UwR/qRqmhc9GhF7G//ylAERflNk9g4kMSWE4Per0pMp8pxM+kS+Y6mkXafI7XdU6bGqFVntnfgl1xr9SF9m1+EXCbzf1oDLecmdxxhjzhmRa4S/1lOlx8/Dv9UR4c9mpxopiEgUwE0AzgdwGIAPichhnmZ7AHwJwA2e5XEAX1PVtwA4AcAXfLYdMvhQhhBSyVDwExuv13wgkbSFU6Yo/6TR1YHrguuwpy/3s/Jk24+Ft7a+a781mSe2yrQvC0so++UrZKrSE0+ke/iroikPv6+lJ8jD7+iXc2bhIAYSyfSkXedxPOepPv1xJu068wEsS89AQvHqm62O9uGrAnmx+iMigU+ATp1jeOLHOCaD8xuEBQ00gvrlnG05qErPVkeEf3xdVcb9jyDmAWhU1Q2q2g/gTgAXORuoarOqLgEw4Fm+XVVfMV93AFgDYHqxOrpqe3uxdk0IIWUPBT+x8UY1E6oOwe//URlTG8OXzprjuw4IFnx+ovq5q87EM18/I3T7oOPEIoInvnoa7r/iZPzxY3Nx0oGTXG2zWXoyiXC/65CtSo93QBCNiEPw+xw/0MOfOvYl82Zi8TVvx9XnHxp47IGEpiXtOk/Nex1UNS3KaZTlNF5XOwZ9QTXyvQND6/Cjsnjenf2JRiRw0HPDxW/FA188xTXgmTmxDn/99DwAqbkTss2EG40IFl/zdvv9Ow6fins/fxLed+x0JJLqm4i9va0XtVUR3Pv5k3DyQZPx+FdOw2kHN2Q9rwpnOoAtjvdNyEO0i8gsAMcAeLEw3Urnk39aUqxdE0JI2cMqPcTGG6hOqtrlE4ME/4ENozGhPlOE33+5n6CbNLoGk0b7W1WyOYCc6yMRsSu0+LbNI8JvLYo5BkB9ZpQ7Yx1+1bR68rGI2Nfaf8Iy/31Zlp5J9dWoq46hrjqGGROCI/2Gpccr6h37i6bOwVrnHexUxyJQM8bv/AwECv4qb85AygrlnD/AT89bM+1m8vDXVkVxxPT0+R1OndOAg6aMxvqWTgD+uRbuYwFTx6USnyMiOHa/CbjzpTeRSKpvKVIA2GdsLY7dbwIAYM4+YzAxINI/guwhfmca/AfhtwOR0QDuAXClqvqG4UXkMgCXAcB+++2Xax8JIWTEU5AIv4jcJiLNIrLSsWyiiDwuIuvMfycU4likeHgj/Kop4RJUiz+bEA9O2s2tbzlZerK0zStp14o+myFr5/Ey6H0zwp/u4bf3m8MMxVaE33kvMl0Xv6RdJ95ypX5Ju856+TUuD3/ICL/ZP+/nJ2Md/ojkJZhjEbEHNGHq5Lv6aT9diCARUKUHQFruhDfBfAQJfYsmADMd72cA2BZ2YxGpgiH2/66q9wa1U9WbVXWuqs5taBjxT1UIISRnCmXp+TOA8zzLrgbwpKrOAfCk+Z6UMV6Jk0yqLYyCqpJks9oEluUsdNKuU0Rnq+iTNWk3uHKO5aN39idTlZ6gibfsY+WQtFsdtcRz6gQznUm/j4ffSbqlJ/06i8Bh6Un38Kf1MeYvpMPMNmsNQARh6vCn4xzcBM0NYfcrTfCn/k34JO1a1ihvOVTvE5Qw1qUKYwmAOSIyW0SqAVwC4P4wG4rx5XArgDWq+ssi9pEQQkY8BRH8qvocjEoMTi4CcLv5+nYA7ynEsUjxSLf0pASld0Ili2z1xoOEW7Yoe9j92OudVXoGGeHPtH9rMOA8RK51+J3CNLc6/OkR/kzXv6c/kebhd+JNbFWkR6hVU4Lf2e8gS4/3XK3zC3PJrfui0JwnCAPcTywylYoF0q+xdR1jkQh6BxI+idbGem85VO81HGmCX1XjAK4A8CiMpNu7VHWViFwuIpcDgIhMFZEmAF8F8B0RaRKRsQBOBnApgLNEZJn53wUlOhVCCKloiunh30dVtwNGNQYRmVLEY5FC4JO0a+mZIA9/Lt569/LcBF225pnKcnrJJ3ps7d/yzB86dQyWbNoLIEQd/gwRfr/zChLx/pae4D7Hk5qbpUc17enGREd+Rk2ICL9fgjIQbiIqy8OvmtuEXRbWxF2xiGQc6ADp+7cGGNGIoLs/gW//e6VrvTWoSxP8nmto3ZsMY8CKQ1UfAvCQZ9l8x+sdMKw+XhZiZE1ZQAghJaPkSbtMxiofvLpVHUm7QQIq13rnYbcLux97fcT5OneP/qJrzkLfQLqIrYoKBhKp6jUnHTgZd33uRPQOJPCx214CkDnCH08k02fazVBC1Oif/76sKPOokB5+Y5tg4eu9p4qU+jr3sH1w2WkH4LB9x/rX4Q8Q/EGDm1hE8MiVp2LJxj347n2rAAGe+trpruRaqzuq6RH+g6aMxp8+8bbAcwFS4ru+JpZ1gOG97FY/gz46VnKzNbGXhff6ZssdIIQQQkpBMcty7hSRaQBg/tvs14jJWOWDtzZJIplK4gyaeCuXCbFy2c5LIS09fvuaNm4UZk2uT1sesyvHpLaZN3uie2bWhNfGkno94FOHP1vSbtC5Wntxeviz5StUxYKvRXqEP9Wfmqoo5s6a6Fpf7arS4z/ISZtV2BL8UcGhU8difF3qicEBDaMxZWzKE28NLpOafg1mThiFmRPrAs/FOIbRv0zzOdj98lx3a5DonETMiTWm8+7be/+ylQMlhBBCSkExBf/9AD5uvv44gPuKeCxSALy1x5MOU3dQpDhfD3+uEf5s44NME2+l7yv8sZ214Z04xa+3ootTBPbFk2kzGDuFtl9XggZDVlTdmU/hV+3GSW9/InBd1Ovh11SStnMsYJ2d8zMQVKUn7q1I5PDGO/flh3XaScf8DxZhBohWQnVdloRdIP0pkPV2d2d/xu28s0p7P2pB1awIIYSQUlKospx3AFgE4BAzIevTAH4K4BwRWQfgHPM9KWPSqvSoI8Kfp4c/SKflKvizRe2dIj4fj34Qljj37tL5xMNrhXJ2dSCR9InwZ07aDRT8psge5UraDe47AOztHghcl1ajH6nz9LNFOa1IoZN2I9aA0UzINQdHft22zlt9IvxhBmlOS0820qr0mO/3dGUW/N76/t57RUvP8CWXQAAhhAw3CuLhV9UPBax6e8ByUob41uE3Xwd6+POM8Odq6cllsqx8KrwEYddZ905I5bge3qRdQzgoqmMRDGTx8Odi6bEi/C5LT5Zzbe0JFvzeakXGAE/S1lkfC+c9CO3hty1h3nr1fgMd63h+Ef6Ak3CQi6XHe92sc8sm+OtqvJYe9/raGAU/IYSQ8qOYlh4yzPDmnhoefneE1kv25MihSdp1ivx8KrwEURWwL6eA9SbtWpvUVUcDIvyS1ta1fcAx9zH97gc1jHZsn/lc9x1fG7guIuI6vuHhT+/jEdPHAnAL6aAI/8GeGY5THv7sXzXWZ8nPwx8G616FifB7cx+sz88BDel5HE7qPZYe7+efHn5CCCHlSMmr9JDywetFT2qqOo0lcN/91n1x/2upiTSzl+UMEPy5RvizHadIQ9dYqnSMa7nTwx9Ue35UVRSt3QNpHv4ql4ffJ8IfcK4XHjUN08ePwikHTXZsH9z3UVVRfPWcg13LvO1jkYhtFTJmVrYsTKmGv7nkGKzZ3o4n1qTy7vs9ybmLrjkLm3d34+iZ413LbUtPCAFvR/ihaU8f+gKeKLjOJZek3YAcgV9+8Ggc9YPHArfz1vf3fr4tDz/dIYQQQsoJRviJjZ+HX+wIv/FReatX0OVY/tAiGqRqA8i3/Odgsb3nacszWHrMf61ob++AO3HW7eFPP2ZgorMITju4wfUEINNpnzJnMmoyWExEfISvPeNsanl9TQxzZ0109dVp6amrjmLauFE44YBJaUmrdtJumPudIcLfMxCcfGxh3atQSbtplh7j37G1VZjnqU7kJD3C715PD//whWM0QkglQ8FPbLwefudMu1ZE22vlyLsOf84R/vxyBQaLJey9or46o6UnFeEHgG5PpZxsHv4gm5TfOWa6LlkTnSEer37KwpUtt6A/nl2AA86kXY+H36+t7eFPP1fvoMkPaxtvYq1vv7xJuyGrPHntQt6mI22m3UrizT3dpe4CIYQUDQp+YuP18DuTOK3o6YDHWpFvYD1XC06+9f4HS1Bk2lnf3mtnt7piRXu90emsdfhzmLsgo+D3Ea6ueyzu81PH/vy2dQ5EgurwB/Uv0wRg3rbJZPrEW95Bkx9W6c8wUfZMZT8zPY2ojmVOPqaHf/jyi0ffwOOrd5a6G4QQUhQo+ImNV8IZnm7jdXXUEDLeCH++QjvnCH+WT2qxIvzRgPrxTgHb0++erMmKao8y7R89HrHqPHe/yxCNSJqHHfC3/2S6jNmSlwVue5EzZ8P/aULqdVDSrhfryUiY+2M18avD772GfliD0nARfu/7cBH+9P34e/gzTL5MypjP/mVpqbtACCFFgYKf2HiTdoGUoLSint5kzXx1dqHr8BdJ79vJpt5L4xTkXR4xaq2pq0pF+J3ddw5e/AZMtVVRjB1VlbY8mwhPa5/lmhzQMDqt/Kb1LtuTh7CCf8CciMsS45PqawAA+09KnzXXipYrUudaF5AH4YeVtOudHMsPb2TeeW1jOTx+sjarjkYQjQhG1zDCTwghpPyg4Cc2qsBJB07C1ecfai+zhJ/lWR9IJPHwl0/FxcfNMNbnqbRzneQm3/KfgyUWkLTr7c+5h+2T1henWHVbWhyWHp+/wK+cMwe//uDRacv9roHfsuNnTzT3HXxNzjlsH3z/XYe5hK7b0pO+jXN/YarmACkLmHX+p8yZjD994m340tvnpLW19u6M8I+tNQY+YSw9qbKcuYtu56Xye7ry/NVn4YEvnuKzndH2vcdMx98+fXyokqCEEELIUEPBT2ySqpg6thZHzRiXts728CeSeMu0sZht1isvltD2ku2JQLGTdrNx+iEN9mvrktRWWzYodSX5Orvqd/1mTKjDEdPT74EfftufcMAkAJmfipx2cANqq6JuD7/DwuW3rXcG4TBYcxA4o+ZnHjrF97ra5+JI2h1nPukIU6UnVZYzd9HtmqnZ59HI9PGjfO+JtVl9TQwnHjhpyP4eCCGEkFyg4Cc2Vh12cUagrQh/zO3ht6KgxbLSeMl2nGIJ/pht6clsynYKPUs81jkqtrhr7zu3C9pfuP75TtxlHiCM9nRF+B3n6Pd0IOKy9KSi8JkOYyX3Bk3c5sTavSvCPyq8eI/ZZTkHZ6vxi/AHYScam9duqP4eCCGEkFyg4Cc2aiZtOoVimoc/7o7YDlVEM9txitWNMDPEAm7R65xp18IZ0XYOqIKsSmEtT+Ijt61Nw9ybmMfSY2l+P9HrjPonkoraWPZrY006FqYOv9VfdRzLsvSEocr8TIZJ2s2ENdjIJdE4NVii4ieEEFJ+UPATG8PD7RWv6R5+ICXg8vXw50q24+Ra9Scs1nlnq7rijtobb0Y5rCVVOVh6vG3CHte7bZhBg7NKj6rauQp+19u7uzAlKK3PSxhrVCrCn/p81eZQ5tKO8Odh6XHtxxL8Ia6fdZ2saRpS135QXSCEEEIKCgU/sUmqpkWMrXdTxhjVVaaNqwXgjPAPTd+yia+iWXrspN3Mit8VtTf/HVWV+vNy1m93+cVD1NzPVFc+08RdYS6JU4cfNGW0Han265f3WJlm8bVoMD83+44blbWtVcFn3/G1qfr9OUymZV2nXGxAE+ur05ZZgyDrdKePD+57w2ijz1PNvwt6+EcuqoqTrn8S/1q6pdRdIYSQNFhSgthYSZtOQWq9PumgyfjDpcfhDDM5NeXhHypLT5b1OQr+R648NZRX2xpI+EX4//GZ4/HhW14E4LVBpUeagyrHBF0+53V96mtnBM4C6qzys+CqM7Gnqx/Pr9+Vcd9+x5k3eyL+eOlcvPf3zwMIVwK0pip7vOAjx++PyaNrcN4RU7O2PWXOZMz/6HE469ApuO7B1WY/Irjn/52IaSEGDO88al80jKnBlDG1WdtaPPzlU7F5t/vaWvkGERHcedkJOMBMUPfjvCOmYv5Hj8XZbzGqNOU6oRypHJIKbGvrxTfvWY6L584sdXcIIcQFBT+xUZhJuz4efgB4x+Ep0RaLDq3gL3RZzkOnjg3Vripg4i0AONJRzcg9SDL+ddpRnBFqZ1+DBirO05k6rtaOIKe1czxZmDmxDjMn1uGF9bvTjhOEJexPP7gB4+qq7BP1rcPv6WttiAh/JCI4/8hpWdtZWAMD67rEIoLj9p8Yatv6mhjOOnSf7A0d7DO2FvuMdV/bqD2YTVU8CkJEcN4R01zvCSGEkHKD8ShiYyftOpYFBcGjQxzhz0axPPy2pcdH8btEvmO5dU2cSa1jHMmn4ar0hDufTLPvhtlHlSf52jpNvwi/V8yGifDni+2jD5HsW6xj5/PZLpe/B0IIIcQJBT+xUTWTdp0R/oCqI1YSZrmUISyWlSJTsqnz1P1EfJVT8Lsi/M62g03azW698cXy6nvKq1oe/mxlOQGgJkSVnnyxfPS5lMgs9LHz0e7l8vdACCGEOKHgJzappF2n5cS/rS0Uy0ThFC3Cb3n4fUw9QdanVMJp6uKNrjUEf3Us4mv/8TKoCD/CR6i91iw7wh9iIFEdwtKTL9Y4q1jJ2JmIDeKzbV/HLFWdCCGEkKGEgp/Y2GU5Q0X4w4nKsaPC11HPhFUec9Yk/wTKYlkpppkVWib5VHPxm6AMAGZNNvpoiXwAGGO+HlUV9bX/pO17EBH+XPZhT55ltk0krch/eluvALbuyQENo0P0NDdKG+E3r0ke245ER4+InCcib4hIo4hc7bP+UBFZJCJ9IvL1XLYlhBBSGJi0S2ySSU0XkIEe/nCWnnmzJ+Lr5x6MGx5bCwB4+utnYE9XX859G1dXhVs/PhfH7jfBd32xnjR84qRZmDy6Gu86at+0dUHi7teXHI1F63fjsGmpxGDLw19XHXVd42DBn3+E33oakfGSiJUU6/bwt3UPAADG16UPcLx9ra+J4k+feBuOciQvFwrrCUOhB3JhPn+ppzq5M9I8/CISBXATgHMANAFYIiL3q+pqR7M9AL4E4D15bDvkvLB+F046cHIpu0AIIQWHEX5iY1Tp8U9A9VKVQ2Lju96aEsuzJ9eHrrri5e1v2QcTfCLtxSQaEVx09PSsAwqnQB9bW4V3HD7VVXvfqtIzqioaKmk3LH7XP5mh0o6XmMfD39EXB5Cad8F9LPf7qAjOPHQKJo1ObztY7Jl5C6yfw3z+rNmVracduTCy5D4AYB6ARlXdoKr9AO4EcJGzgao2q+oSAAO5blsKnm/cVeouEEJIwaHgJzaqhk0lqPqME2+yZyZGQtQz2xladfhrPZaewZZx9Ns8aRrIQ820GzBj8pSx6WVAvfexmPkbpfzI2BH+PEL81jUfAR95i+kAnDNNNZnLir0tIYSQHKDgJzaqanj4HcuCNJ0VBS2XpN1S4JeoG4S1vq466q7DP8jL5yfqLaEaRnTGbA+/u/Hk0T6WHk9nh8JfH5RDUkyswWw+Ef4RiN8NCnvhQm8rIpeJyFIRWdrS0hK6c4QQQgwo+IlN0p5pN7UsKEqcS63ySh0UOMVotstgXadR1V5Lz+Cuja+HX0N4+E2CntTU+FTg8bYp5n0tZZUby06UZKmdMDQBcE4rOwPAtkJvq6o3q+pcVZ3b0NCQV0cJIWQkQ8FPbBRG0m4YIZsq55h9vxWq9z3VjPzZf1IdAGB8XZX9PkzSblgyefj9ouMNpjd/nFk9ySodGiaS7j1WsUqhOimFNSY6CEvPCGQJgDkiMltEqgFcAuD+IdiWEEJIDrBKD7FRvwh/gBCM2ZMT5S4UhwPPfP0MbG3tydjG7cX3b3PX507Ea1taccpBk/G/F78VFx41Db9/Zn3W7cLit73aSbvp675w5kGYMaEO7zpqGoCUh9+KZj/0pVPRM5DwPVZa0m5RI/xmHkLRjhCM9fSKEf7sqGpcRK4A8CiAKIDbVHWViFxurp8vIlMBLAUwFkBSRK4EcJiqtvttW5ITIYSQCoeCn9hYSbtOgj38xoowom8Y6n3Mmlxv19MPwj2Blv9J7jO2FucePhUA8D/HzTDbptYXJ8IfnLRbFY3g/WY/gJS4tfzqh+07Nm2boGMNxUCuFJ8dazBLwR8OVX0IwEOeZfMdr3fAsOuE2rbUZLvt9y3bii/fuQzLvneOb/laQggpR2jpITYKTRP4gZaeHKr0lCLxcqjJ5QwLmrTrs8yOjufg4Y+HSFBNs/QU8dujPDz8pesDKV9uW7gRALBpd3eJe0IIIeGh4Cc2fkm7QVFcu0pPKEtPQbpXduRbXtPZcrC2GL/rrxnWeamya84nsx8r4n1f2VV6CCGEkEqBgp/YGGU5QybtBpRz9GM4evjDECZpN+t2RbT05FKlZyCRR4S/iPe1lMH1QpQbpRuIEEJIOUHBT2ySagjXMIK0xpxFtjo6kgW/w5qTw1/SYEW+a18+x7Wr9ISZeEvC15xPt/RUpoc/msvNJBXHxl1dpe4CIYQUHCbtEgBO37eEilyPr6vGrz94NE47OERN7CESbX/99DxM9ZkhdigQCO79/EmhTjWsiP3zJ9+GmRPrshw3nWQeHv5yE/yljJBbJVQHQ4WOcUcEq7e3l7oLhBBScCj4CQD37KxOS0+m6Px7jpkeat9DZYk+dU4JJ+QR4Nj9JoRqGvaJxxmHTMlvX5phnQdvlZ7Mx3K/L66lp3RlOaeYcxWQkcmujr5Q7ZS+LULIMILPrgmA4ETPQmi6SrX0OMnJw1/A42Ysyxlieyv5OlSVHo/iH5IZlEvw2Wmg4B/RdPX7z0NhMwK+zwghlQcFPwHgFon5JqMGMRIEfy7nWMjLkXnirVwi/CGq9IwQS09dNR98EkIIqSwo+AkAr6UnRSESTEeA3s/pHAs5APLbVVKD13nJpUqPd3/FtPTYxyz6EQghhJDKh4KfAHB4pr1JuwVQXCNC8JdImvrX4Q+eadfLuFFGguqY2tyj2sW09Fj9yadfpaS2yvhKnVjPGVgJIYSUD8Pr15QUDXVFhcMl7YZlJFh6cjnFQpbl9BX8tqUn+/bvOWY62nsH8KF5+2Vt691diIqsefPxk2YhGhF89IT9i3eQDNzz/05Cfzy7zcnLkdPH4br3HoF3HrlvEXpFhoolm/bgbbMmlrobhBBSMBjhJwDcvm96+HMnlzMsZGDcb1+pibdC1OGPCD558mzUVkVzPnYxPfxV0Qg+efJseybgoea4/SfgxAMn5bydiOAjx++PcQUo7UlKx8XzF2Vtwxo9hJDhBAU/AeBJ2nUsL4Rto/Llfm5R+0JeD7/j5jLT7mAYkio9hJQZ/NQTQoYjFPwEQIaynAXY9wgI8OeWtGsK5WJdF7vCZpEvfIyCnxBCCBkWUPATAO7ZWZ1R48JU6al8YViqOvx+5OLhzwXvbRwJVi1CCCGkEqDgJ3h01Q4c/cPHAJhVehzrCqnpxtRUbo54ToMas20h5fIohwffqmxTV527Lz8XiunhJ4QQQkjhqFwFRkKzcmtbqnY73CK/UJrulx94K47db0JhdlaG5HKdrLaFevLxqw++FUfPTF3br5x9MCaPrsa73zq9IPsPgoKfkBRqPloLMWk1IYQMORT8xLbzAIYYdcb4C1Vf/n3HzijIfsqVnAL8KGyE/73HuK/tqOooLjvtwALtPRhaeshIxjsb9H+WbStNRwghJAS09BAkHOXGvRNvMYgblhyq9Ij73+EKI/xkJBL0d9vc0Tu0HSGEkByg4Cf2o2jA58dsuKvSISKnKj2W4B/mBf4Y4SeEkOLwn1e3YtW2tlJ3g1QQFPzEZenx+sop6cKRi/gd7kLfghF+QshQ8HzjLiRGWHLElf9chgtvXFjqbpAKgoKfuJLM0pN2KerCkFNZzmF6Sb2z3pZoElxCyAhi4bpd+MgtL+KmpxtL3RVChjX8ySauyElExFOHvxQ9Gn7kVpXTaDzcIuRHTh+Hb11wqP0+GuHXByGkuOxsN3IjNu3qKnFPCBne8BebpHn4nTJ0mGnSkpGLTcdqGYsOr4srIrjstAMx1qzzH+VokAAQkfNE5A0RaRSRq33Wi4jcaK5fLiLHOtZ9RURWichKEblDRGqHtveDYWRZTAghw5uiC/5sPwak9DgtPRFxR6tHwiy5hSCnpF3zr85rkRkuWP1mgJ+ISBTATQDOB3AYgA+JyGGeZucDmGP+dxmA35vbTgfwJQBzVfUIAFEAlwxR1/Mm6E/dW6aTEELKiaL+ZIf8MSAlJuGM8Jv/s99T74cinzr8w83SY2E9mWCEnwCYB6BRVTeoaj+AOwFc5GlzEYC/qMFiAONFZJq5LgZglIjEANQBYDF7QggpAsWO0YX5MSAlJs3S44zwV0hFmWKTk6XHbFo1XAW/GdofrgMWUlCmA9jieN9kLsvaRlW3ArgBwJsAtgNoU9XHithXMgzhgxNCCkOxBX+YHwNSYpKeibecUNOFI5+k3dgwtfRUxyxLDz8cxHek69Vovm1EZAKMANBsAPsCqBeRj/oeROQyEVkqIktbWloG1WEyTOHXDSGDotiKI+uPAb/IS4+zDn/Ek7RL10Y4crP0GAy3pF2LWISWHmLTBGCm4/0MpNtygtqcDWCjqrao6gCAewGc5HcQVb1ZVeeq6tyGhoaCdX4ksGpbG2Zd/SBe29Ja6q6QPIknkuiPJ7M3JCQDxRb8WX8M+EVeehLemXZZhz9ncrlOVtuqYZr1aj2Z4GeDAFgCYI6IzBaRahhJt/d72twP4GNmtZ4TYFh3tsOw8pwgInViPPZ6O4A1Q9n5THT2xTOuHy5Juk+/3gwAeGz1jhL3hOTL+b9ZgIO/83Cpu0GGOcVWHGF+DEiJUdfEW0Lffh7kM/HWcPXAV5lPJhLDRfGQoqGqcQBXAHgUhli/S1VXicjlInK52ewhABsANAL4I4DPm9u+COBuAK8AWAHj9+jmoT2DYCyh7IWVy4YW5fcM1jV3lroLpAKIFXPnqhoXEevHIArgNlVdVcxjktxJZkjaZRQ3HPlYeqpiwzTCbw5U4gk+YiaAqj4EQ9Q7l813vFYAXwjY9vsAvl/UDpKKgIEoQgZHUQU/4P9jQMoLZx1+EaGHPy9yqdJjWXqG58W1LD0DCUbeCCGEkOHA8AwxkoLxp+c34r+vpdIqjIm3HHX4S9GpYUgu2n24W3qsmXaVBfNIBdPdn9nDT8hQcO6vni11F0iFQME/wvnhf1e73nsd/Cy9GI5cfL22pWeYluX86f8chS+ceSBOmD2p1F0hpGgkK2w8m4sV/pt3L8fHbnupeJ0hoVm7k/59UhiKbukhwwuvvqfcD0duSbtWHf7heXUnj67BN95xaKm7QUhJGS7jgXySjP+5dEv2RkPEcLnOhJQ7wzPESIpG2ky7NPGHIpfLZCVJx4ZpWU5CRjL8RiwN/CkiZHBQcRAXRtKuw8PPL9lQ5FLNKGF6BaqGaYSfkJEAq0ESQioJCn7iQuz/M2BZzsIzYJazHK5Ju4QQQggZXlDwExcREbelp3RdGVbkMi6KJ6wIP//8CKkUOEEUIaScoeIgLkTcIp8R/nDkkusQTxoR/hgj/ISUPTvaeitCzA/bMxi2HSekvKDgJy7EU4efIf5w5KLdrQmrYozwE1LWNDZ34ITrn8QtCzamrauAMcCwgj9FhAwOKg7iwhupZoQ/HLlM+x43PfxM2iWkfFEotuzpAQA8v36XvZxfiYSUHlXFpl1dpe7GsIKCn7gQuCMp/G0LR04e/iTLchIy3Iknkjjnl8/i6debS92VioYzehM//rhgA8644Rms3NpW6q4MG6g4iAtv0i41aThyGRjZgp8RfkKGLS2dfVjX3Ilr7l1R6q4MGxasa8FLG/fktI1lneKTlcrhyTU7MevqB7G9rSfvfby8eS8AoGlvd6G6VfFQzhEXRtKuow4/Y/zhyGPirWp6+AkZFowEv/7Lm/di1tUP4sUNu4t2jEtvfQkf+MOivLblb1HlcMdLxkzOK5oYnR9KqDiIi7SynPyODUUuuQ4fP3EWPnL8frj8jAOL2CNCyGBQRU4D+XIfFGTr3wuNRp7CgnW7MjckhOTMjx9YXdTBdBgo+IkL7+9bLuUmRzK5XKX6mhiue++RGF0TK1p/CCEEYNCGkHLgloUb8cGbF5e0DxT8xIVw4q284MCIkMqiuz/uu9xrLWFSKSFkOEDBT1x4dSvLcoaDV4mQyuInD72OJ9fsBFDauZ+6+uJ4x6+ew2tbWofkeOU2gCmv3oSjo3cA/3l1a6m7QfLk8dU7KzK/gIKfuDDKcjqSdqlkQ8GBESGVx98Wv1nqLuC1La14Y2cHfvrw60U9Trl/hZV7/5xcfe8KXPnPZVi1rfJE40jgs39Zinf9dmGpu1FwKPhJGq6ynMPoS7ak8DoRUrH0xxOl7sKwZ6jLJ97+wqaSTcy0vdUoN9k7wM9NsSn3ZPlygoKfuEiqV7tSyYZhOEWfCCG5sXhDcO34ne19mHX1gxgwZ9AuV4pl1Wls7kBLR1/Wduf/ekFRju9H70AC379/Fd4/P78SoKT8YZnW3KHgJy5U1ZWAygh/OHiZCBnZdPT5J/mWGksY/eHZDUgmCy/6z/7lczjpp09mbVeK69PeO1C0fb/y5l708ckPGUZQ8BMX3tLTrD4TDnr4Cal8uvrieGlTbjPFFopCROi9onvLnm7s6eoHkPquz8ciMZAonq+iHCwbqoqfP/I61rd0AgA27urC+373An5w/+oS96x8efqNZsy6+kG8vqO91F0hJhT8Ixj1+SZNqtLDnwfU+2SkIiLnicgbItIoIlf7rBcRudFcv1xEjnWsGy8id4vI6yKyRkROHNre58abe4J96I+u3FGcgxbxu+XUnz+NE37ijs6Xgb72pRTfsbs6+/Cx217C6zs68Ltn1uNjt74EAGjtNgZJq7eXh5h9bm0Lmjt6Xct2d/aV1Gb22CqjwtXLm/eWrA/EDQX/CMbv6a53GYVsOOgnJCMREYkCuAnA+QAOA/AhETnM0+x8AHPM/y4D8HvHut8AeERVDwXwVgBrit7pAuF9+rmtrTegZXnTX+a5B1627OnG2p0d4RoPcvRyy4KNeG5tC+54yajWlAhpiRrqQdPHbnsJH3DkK/THkzjux0/gW/euGOKe5Ieq4scPrMbyptZSd6WioeAfwcST6V/0SY+Hn5aecPAykRHKPACNqrpBVfsB3AngIk+biwD8RQ0WAxgvItNEZCyA0wDcCgCq2q+qrUPY95wptb1kRVMb/vT8xtJ2osSc+vOnce6vnsvYptTfx6nPydB1ZNPu1NMnaxD30IrtQ3b8wdCfSOKWhRvx/t8PfZL1cdc+XvSSt+UCBf8Ixkfvp4UmqGPDUeofGEJKxHQAWxzvm8xlYdocAKAFwJ9E5FURuUVE6ovZ2cHwnpuexwU3Fq/SzLItrRnLOKoC7/rtQvzwv7n5xl3fTSEGLKUe1ADAva80YUsG+1Q50tzRm5bEW+m/C3e/3IRZVz+I3Z3ZqzQVi8F+XHd39WP+s+sL0pdyh4J/BBMU4XfCZNRw0NJDRih+H3zvb3BQmxiAYwH8XlWPAdAFIC0HAABE5DIRWSoiS1taWgbT37xZ5pnp1i8HyqK7P7eKNDvaevGem57H1fcsT1tXyO+WTIm/1lf9wsbU9V24blfBjp0LX73rNbz3d88DKI+Zfx8x8zMy9WXedU/iC39/Zai6VHISScXX//UaAGBzCQZn5ShN/rp4c1mX6KXgH8H4+RG9v2Hl+EdVjvA6kRFKE4CZjvczAGwL2aYJQJOqvmguvxvGACANVb1ZVeeq6tyGhoaCdHyw3P1yU+C6w773aE776jDLR67cVpgk0OaOXuw1q+/kysqtqT589NYXM7QsLrs6jf63dlulNXP/ku1PJO3KOoOh2ZxnwPp9DJL9T6xpTlt2xT9ewYnXZy9bOtzY253f58vJZX99uSi+/fuWbcU2c/IzJ9/412t2PkYx+PkjhjWou788y7VS8I9g4j6C3xvhZ+Q6HLxKZISyBMAcEZktItUALgFwv6fN/QA+ZlbrOQFAm6puV9UdALaIyCFmu7cDGDZ1Dtt7Bl/j/b5lW22xX0jmXfckjrn2cd91ze29uObeFeiPl2cU0ssvHn1jUNu/57fPF6gn6YT53n9g+XZsH6YJ3fnQ3NGLp19PH/gEcfsLmwd1vPXN7gFdfzyJL9+5DB+8OT0f4F8vN+GaYZLIXAwo+EcwvhF+z3uW5QwHrU9kJKKqcQBXAHgURoWdu1R1lYhcLiKXm80eArABQCOAPwL4vGMXXwTwdxFZDuBoAD8Zqr6Xmtd3tOPLdy7DVXen23j8KISx5cN/fBE/+O8q3PHSm3hizc4C7LE8cQaqijHh13NrM9vKhsKE1NUXx01PN2asHFQKM9QH5i/CJ/+8ZMiO97+Pr3W9t2xXO9tKl1dQrsRK3QEytNz/2jbEIoILjpzmG+FP86VSx4aCep+MVFT1IRii3rlsvuO1AvhCwLbLAMwtZv/KFeux//a23ozCbDDfLX9ZtMklCFdvb8f+k+r8j8Mv+6wogNe2tOLXT6wL1b6YV/Rnj7yOvyzajJkT0+/nLtOCVApribNaUL6s3NqGhY27cPnpB6at29HWi7YCPF0rKqVPO/GFgn+EcevCjRhVFcEFR05zTbM+dWwtDt93LE4/eIqrPSPX4WD5UkJGFq81tRV8n4X+Fvnefaswqb463LHL/Cssl/4V61xUQ3rXcyx19ML6XRAITjxwUsZ2m3d3Yf9JRiGrTvPJhZ81a/nWwn828yXbpfBLhH7n/y0EAF/Bf4KZD3H+EVMH37kCU+Z/QrT0jDS6++J2xMcZ4Z8xYRRu/cTbMKo66mofo6eHEEKKRrGDgR29/paWzw+zijLD8ZcobCDow398ER/64+Ks7Tbu6hpsl4aMfAZd/YkkWjqGvxXHO4jpcTxpeWxVkWbkDgEF/wijqy9uC/2Eoyxn0I9OlIKfEEIKjqs8fhjVH6LNV+9ahllXP5hvl3LiE396CTc/V571ywv1qzXUJUF3d/al1fLPhy/d8WoBepOZYimDF9aXphRsIQga4P34wVQtgsv++vJQdScNCv4RRld/wjfCHwQFPyGEFIaVW9t8Zz+1hKWfXhBPm0zc+8rWtGX9IWuC5/pN/8wbLfjJQ5UxQ2kiqfj+fSux1aeUoxstqn3zuB8/gc/cvrRo+3fy1Os7ccJPnsw42duQkOeYqpwtaN4B/J48S+QWGgr+EUZ3fxzxhCn4E6lPZdAkMhT8hBBSGN75fwvTrTSO715n4mxbzwBmXf1gxnr/+dBehDKg5cRvnliHNds7ctpmyaY9uH3RZnz1n8sytgs7T81gngssWLfLlV+XjUwTwAHAx257yXf5tQ+swY72Xt969X70DiRw+PcewSMrjQFrueatlXKm6DK9JDYU/COIvngCAwm1I/zOmvu09BBCyNBhCabXmtqwwkwAdto5mvYa1U7+VWDB/3zj7oLub6jwiqlfPvYGXt68x7Vs9bZ2/OqJtXjP754P3M6PsCIxVy2Z76/nAd96yHe58/hhqyplKyHqx4qmNjR66ttva+1BV38CP304/akOVcLwgIJ/BNHdZ/yYxE3vvtPSE/SFx6RdQggpLt/5z0oAmUsaOmfALRUL1rXY1WFKzY1PNeJ/fu+eXOmCGxcAcM8xU+iIbyl/EXuGqMzmu367EGf/8lnXsqfMybQKUXbTiff2DPZ+hY2yD1RIgnAuUPCPIKwv6lTSbhgPPz8ihBAyVMQDPPc9RfZaZxJKj63agfuWbcWlt76Er921bNDHmnX1g6GtJJVCIbzy3zUHhqXgzT1uoe/8uDy8snCVZx4p4L6c7O50i/ur7l6Ot133RFGOVa5QzY0grEk4fD38AdtEy92URgghwxzn92+YYgoWW1t7MPfHT2Dz7uKVa1y2pRWX/fVlfPnOZQCA9S3Zj9Xc3oufPfJ6Ri/6mu3+TyycnvT7lqUnIZeSbH75VLv0ZTc93ejbNnuicIrdPsmfr7zZGnp7J83tvaHbDiSS+NfSYGvZzc9tyKsPFk570iNFKlvZ65mv4OGV6cnzee13IIHFGwybXCrB3k0p8wqcUPCPILr6jQh/wi/CH5S0G6XgJ4SQQuP8ZvWbPCkM/3l1K3Z19uGfS7YUoD/+3/Xvuel53+WZ+Prdy/H7Z9bjpU17sjeGIaRXbUufLMoaZAw1uzr7fMW9Iv8JwIIi/Of96rkce+fmjpfedL0/5DsPZ93mhcZd6MrBHjT/mfV5P2HyXsWVW9vwxJqdee2rHPnOf1bikpsXY0NLp52X4/3sJMpE8VPwjyC6PJaeeJg6/IzwE0JIQbngNwuyWlrCJmWWI32mOMykc5zr7lyyBRfeuBBPv9Ec2H6orsea7e2Y++MncMdL6YOo1u4B7O7MXmIxl/r9HXnmRAT9NPeFGDyu9BlcpbZPF/a7OnP3ugfdLWsWXYswT03+8+rWtPr8hf48vJlnbsIbO4yKUJlyWx5fXR4DHAr+EUSXmbSbSCZxy4INWLczlYUf9DfHKj2EEJI7W/Z049oHVvvaWlZvb8eNT/lbPCwhlynW8qU7XsXrOwqXxPvaltZB1+u/a2luTxnuXJKKTL9u2ns2F3Em2f+8uhWzrn4QW/Z0I5FUvLalFX98boPLUrO8qQ3n/8ZI/H1+/S7fSNiVPqU7vaLVSrB2itJyLWPp5ZDvPBKq3VCezpX/XIYP//HFUG2DtIy3u70D7s/7BTcuQHd/eSSkF4tYqTtAhg4rwt87kMSPH1zjWhcUkaDgJ4SQ3LniH6/gtaY2vOfo6Thyxri09UEe9jDc/9o2vLGjA9PG19rL8onCWlyUg20nSOhddfdyfGDuTADhylc+sSY4ml8MLKH+xwUbMGVMDW54bC0A4J5XmvD9dx0OoLiJ0eX0S7qjbfDVaQoZYW/uCJ9PEJoM3fOrdtTZF8dh33sU//jM8TkdxtJOA4mkPcHW2p2dOHF0TU77GQoo+EcQ1ui11+eRHSP8hBBSOCzrZK6R0LBC6o2dHXhjZ2qCqY7e8otOFjIK7Gc1yacPjc2dLlvOlj3BVo5tbUUQomXAbc9vzKl9IRzoNz+3HqOq0yWnAjjlZ0+H2keYyoK/s5KjM0ySlslyFTbvxGLtDsMp8YdnU4nLH/rjYiy46kzMnFiX076KDQX/CMJK0vET9xT8hBBSOCxv7zv/byE2/fTCnLZ9c3c3Vm4N9ll72dPVj9bu7N7y4Uyh8h5fWL8bJx4wyX6fKXn1v69ty9onVQ3XtzL/KVVV/Hd5YSrX+PGTh9In7MqVoEnEWrv7Mb6uGgACrXJhefr13J48WVa459a5+9baPYDG5mYs3ZzbAKKY0MM/gujKIzmIE28RQkjuOMtrnvW/z+DBkGJKBDjtF0/jG3cvD32sO5dswXt/90LOfcyHME8gNoQo3VkofvPEuozrVYGHV7ivfVeBvNrLtrTiW/9eUZB95Uqhf5mfXduCL93xasH2V4wJ2oIi/O09mY/VF09gIB5uxPhaU/iBdjY++ecluOnp9QXb32Ch4B9BWEm7fgT9KUSGSaIRIYSUKxtauvC1fy0L1bbcv3HD/CRY+QRW07bugUEfN+g36ldPGF78TBNb/b+/v5Jl336PvcP1y6+aj4XzWhW6qkx3gfINzvrfZ7Fo/W609Qz+Hjm5b5nxdKRon2efHa8IEOtH/uAxnPaLlG2oGFUyh0NVLQr+Yczyplb851X3xCR3Ld1iJ4Nta+3BLQtSvrJMEf6g0liM8BNCyODxVgUZzuQyaRMA/PxRfztHY3MHXnlzbyG6FLrKUFgeXJGfveWzf1lqv162pdV+XejYWdgnRmEIqrDU3N6Lj9yyGK1+A7YCnY81MPDjmnvDP+VSVbzv9/7J5/nOcxHE9rYeLFznKRPquR65lGYdKij4hzHv/u3zaSXCrrp7uV1W7DO3L8WPH1yDpr1GUlLQY8zxdVX4wbsP910XoeAnhBDi4Kz/fdZ3udfzbpWidM7q7uTsXz6H9xXAivTXxZtzHoQUEufZOWuuOyfFSqrisVU7Qs/Wmw/xEIOeoNwQv2794bkNeL5xN+73yWUYiof/GZ+eePryr5ebMBDwOSs05/9mAT56a7gyoeUEBX8F09FnjMot31t3QHLSE189HSc4kpgIIYSUhuFQrz3In/3FPD3gzR3Zy0Rm0snf/c9KfOb2pcENcthXPnz0luzi7++L38Rlf30Z/351K9Zsb8cP/7uq4OL/oG9nn2X3b4s3py0rRD92d/bhpw+nP8kZqjj31r2eiewy/BkN9k/M74lHkL4qJwYl+EXkYhFZJSJJEZnrWXeNiDSKyBsi8o7BdZPkg+Ups/6Wg76kOZsuIYSUB0+sKY9ZOYMoxoDk4ZU7Br2P1hw86Fv25jerahCLNuzO2sb6/W3u6MNHbnkRf3p+k123PVd2lvBpRhDf/vdKzH+28AmqQdWnXt+RKkn7SIbPj981Pux7jw6+Y8OQwZblXAngfQD+4FwoIocBuATA4QD2BfCEiBysquU/BKogrO9la4QdNItcNErBTwgh5cDn/vpyqbtQMHIdG/zgv6uxo90/2p/NE51LkDotQlt+duuMbGvtyd4oR/yur9/TgCAKnUNhcfnf/P8eGps77ddLN+/BIVPHuvsTT+LPz2/Me1Dlpa1nAF/55zL87H+OCtW+iM6tvBmU4FfVNYDviP8iAHeqah+AjSLSCGAegEWDOR7JDeuuWI/rgqr0MDGXEEJIGLL9WiQdpRMv/+vLmFBfjeP2mxB6/5mixE0FjswXm3J5eL5xVxdmT67P2MZPoPYVKNk1zIRZQWzZk//g5gf/XZ33tl7uWrIFT73ejD8U4SnGUFEsD/90AM5siyZzGRlCrIGYwqiGsHFXF/y0PSfXIoTki4icZ1o3G0Xkap/1IiI3muuXi8ixnvVREXlVRB4Yul6TYvGjB1Iia3dXPxqbOwtSk10g+MeLb2ZvWEb4lWp0LslHBp//mwW4++WmnLY584ZnMq7Ppx//++gbrveZVMSvHl+bxxHCU6qSmB29hS1lWmyyRvhF5AkAU31WfVtV7wvazGeZ72dKRC4DcBkA7Lffftm6Q3xQVV9fZSrCD/znVSPL/qQDJ2Nho7ucVCzC3G1CSO6ISBTATQDOgRHYWSIi96uqM7R2PoA55n/HA/i9+a/FlwGsAeB+Jk/Kkr54ZmfuPT5iNN8Sl06yWXoGU0e+p0A17cMgMriKlmu2t9ult3Ph8dU7MbY2WPLlakG5fZHb7pPpaUaxZ5sVwZB5aJz66cgfPDYkxywUWZWeqp6tqkf4/Bck9gHji3+m4/0MAL7FVlX1ZlWdq6pzGxoacus9AQAEPi0Ta70inlRURyO4eO6MtGYM8BNC8mQegEZV3aCq/QDuhGHpdHIRgL+owWIA40VkGgCIyAwAFwK4ZSg7TfJn/RDOoutEFfjdM8WxU3w6hwo/g+X/nmy0Jwkr9GRXmfjsX5bigzcvLuIRiiMktjryFbYHJCsX0zoVTyRdJV+dycKZWBFQ/tTi4O88POT2oGKFdu8HcImI1IjIbBiRnZeKdKwRTzxp+Oy8pbUijhrISVVEIun2nWhEhkUZOEJIWRLGvpmpza8BXAWgcmalGuF0FMC+48dQiuNi0tEXR5dZwvGljcWNfPvhFwh/+vVm/DWHBN1S8d3/rPRdXkxLz48eWI15P3ky58/fdwL6CgCbdnWhP57E9T5lTIvJYMtyvldEmgCcCOBBEXkUAFR1FYC7AKwG8AiAL7BCT/GwEmK8kX7rTyCpikRSERVJS9Clf58QMgjC2Dd924jIOwE0q2rWsjQicpmILBWRpS0tLfn0kwxznnq9udRdqFjae+OuWYHzIShu+HAB7FzZeHDF9qIVWnpyjfG5K0QeisUZWXIqisWgBL+q/ltVZ6hqjaruo6rvcKy7TlUPVNVDVDX7bBAkb+K24Hd/5K0/wHjSFPwRQdTj12eFHkLIIAhj3wxqczKAd4vIJhhWoLNE5G9+B6H1k5DyJkhJrN0ZzgIzWMqxDGa5wWzNCuD+Zdtww6NvpCUeWY+5Eg7Bzwg/IaSALAEwR0Rmi0g1jPlX7ve0uR/Ax8xqPScAaFPV7ap6jRkwmmVu95SqfnRIe0/ICGMgOfTuuVJV0Rksv3j0dVcOwXBnsBNvkTLA8oqdcMAk13Irwp9IKhJqRfjdf3iM8BNC8kVV4yJyBYBHAUQB3Kaqq0TkcnP9fAAPAbgAQCOAbgCfLFV/CRlKsiVuliIqfe8rW4uy30pMBbzp6eFbc98PCv4Koj/hnyaRSCqSSUVEBLEoI/yEkMKhqg/BEPXOZfMdrxXAF7Ls4xkAzxShe4SQAhP3mVW31FH8XOcmKBc+esuL+Ntnjs/esADQ0lNB9Me9Hn4/S4/7llPwE0IIISQs1z6QPoNtUIR/qJ5i7Ago2VnuWHX939jRgVlXP4iVWZ4KDQYK/gpiwDPqtv7+EmpYeiKSbukp9aicEEIIGYlkm0ysXPFOupWNRRt2F6knQ0exLUuPr94BAHh4ZfGqGlHwVxD9cY/gtz38SSQDknaH6xcOIYQQMpxp7ynOnAXlxO6u/lJ3oSAMVXC0mHkDFPwVRL8jwt/Y3IFV24zpt+MJRULhm7QbOEsvIYQQQorGzx4Z2omXismSTXtL3QWSBSbtVhBOS8/Zv3zOfp1UK2kXGFUdDdyG5M6Hj98PjTs7S90NQgghpGTs6uwrdRdIFij4KwivpcfCmngrFomgYUyNa13fAAX/YPjJe48sdRcIIYQQUkSK6eHXIcpspqWnghhI+H9oEklFPKmIRARjatxjvN64fylPQgghhBAC3LpwY9H2ffnfXsYNj62137/6ZnHsURT8FUSQPSeRVCRVEY2kSnVacDpqQgghhJDS8Oiqna737/3dC0U5DgV/BZFJ8CeSimglToVHCCGEEEIyQsE/DHhoxXY0d6Qmlejqi+NfS7ektevPEuGPcJItQgghhJARBwV/mdPZF8fn//4KPnHbEnvZD/+7Ct+4e3la24F4gIdf3RH+z512AKpjmW/94fuOxfuPmzGInhNCCCGEkHKAVXrKnIRZKH/Lnm572c52//JX/Qn/BFzL0mNF+K+54C24/PQDccy1jwce98EvnZpvlwkhhBBCSBnBCH+ZYwn+pCO7NsiZExThjyfMpF2Hh7+uJurblhBCCCGEVBYU/GVO3PTlO6V8JCD5NihpN2lZehwjhZoYBT8hhBBCyEiAgr/MGTAj/M7ymUHJt0FJu/GkIqFwCX5CCCGEEDIyoOAvc6wIfzyZxD0vN2Fbaw8WrGvxbZu5LGeSgp8QQgghZATCpN0yx5o9dyCh+Nq/XsvYtj+eSfCnW4FmT67HMTPHF6SfhBBCCCFk8Gxo6cQBDaMLuk8K/jInnvQX8X5YgwMviaQimTRm2nXy9NfPGETPCCGEEEJIodnR3ltwwU9LT5kTDxDxfmSaeCuhSksPIYQQQki5E176hYaCv8wJ8uX7EWTpiZsR/qDqPoQQQgghpDwogt6n4C934snwtz1jWU5G+AkhhBBCyh5lhH/k8ML6XdjR1ptThD+obTxh1uFnhJ8QQgghpKzRIsT4KfjLlA//8UVccOOCnDz8QUm7STUtPYzwE0IIIYSUNYkc3B1hoeAvQ9R8lrOnqz+nKj3BHv4k4klFjIKfEFJgROQ8EXlDRBpF5Gqf9SIiN5rrl4vIsebymSLytIisEZFVIvLloe89IYSUH/99bXvB90nBX4Y4I/VBUXv/7YKq9BhRfkb4CSGFRESiAG4CcD6AwwB8SEQO8zQ7H8Ac87/LAPzeXB4H8DVVfQuAEwB8wWdbQggZcWza3VXwfVLwlyG98YT9OjdLT5DgT9LDTwgpBvMANKrqBlXtB3AngIs8bS4C8Bc1WAxgvIhMU9XtqvoKAKhqB4A1AKYPZecJIaQcSRYha5eCvwzpG0gJ98JMvGX4wVilhxBSYKYD2OJ434R00Z61jYjMAnAMgBcL30VCCBlesErPCKEvzwh/kIc/kUwiqWAdfkJIofH7UvF+aWVsIyKjAdwD4EpVbfc9iMhlIrJURJa2tLTk3VlCCBmpUPCXIX3x/CL8QTPt9gwkzAj/oLtGCCFOmgDMdLyfAWBb2DYiUgVD7P9dVe8NOoiq3qyqc1V1bkNDQ0E6Tggh5Qon3hoh9A6kIvxOm86R08fhqa+dnvP+dnX2I8GkXUJI4VkCYI6IzBaRagCXALjf0+Z+AB8zq/WcAKBNVbeLiAC4FcAaVf3l0HabEELKFy2CpydW8D2SQeOK8Dui9rVVEUysr855f80dvUgyaZcQUmBUNS4iVwB4FEAUwG2qukpELjfXzwfwEIALADQC6AbwSXPzkwFcCmCFiCwzl31LVR8awlMghJCyoxgefgr+MsSZtOuM8NdWRVFbFfXdpioqgUm7ze19iDNplxBSBEyB/pBn2XzHawXwBZ/tFsLf308IISMazrQ7QnCW5ezoHbBfxyKC6gAjftByIPXEgIKfEEIIIaS8kSLEQij4h5hV29qwva0ncP36lk78ddFm+31rT0rwR0QCffjVsey3kpYeQgghhJDyphh1+GnpGWIuvHEhIgJsuP5C3/XX3LsCL23cY79vcwj+2ZPrA/dbVx3D3u6BwPUAmLRLCCGEEFLmJFmHvzLIdCN3d/a53rd2DyAWESz73jn49oVvAQBs+umF+MRJs1zt9hlbk/W4tPQQQgghhIw8KPjLDG+Uvq1nALGoYHxdNcRhyampct+6aeNG+e7PKfJp6SGEEEIIKW+KUZaTgr+M6I8nsaer37WsvWcAVZH02zTKU61n6rha332Orkm5tmjpIYQQQggZeVDwDyHxgJlwLXZ57DyAkbQbi6YLdW95zmkBgl9VMcYU/T67IYQQQgghZUQx6vCPCMEfTyTxwvpdJTl2XzyBFzfsNl+nC/4NLZ3YsqcbXX1xPLJyR9r6PV39vt77sBF+BTBptDFZFz38hBBCCCHlDevw58lvn27Eh//4oqv6zVDxkwfX4IM3L8banR2+gv+s/30Wp/78aVzxj1fwowdWAwAOnTom637nzZ7oej9ljFvw23X5FTh2vwlGm7H+gwJCCCGEEFK5jAjBv665EwDQ0pFumSk2b+zsAADs7uxHn2NCLS/PN+62X//fh47B69eeZ7//w6XHpbV/y7SxaLzufNuuU1+Tivgv/OaZWHj1mQCMCP8vLn4rXvr22/GOw6cO6lwIIYQQQsjwY0QIfuvJSDEekWQjZibcxpNJ9A64I/xOT3+/4/Xo2pjLoz9zQp3/vqMROxG3rjqVnBuLROxEX1VFNCJpTwAIIYQQQkj5wTr8wxDLNx9PalqE31uRx8Ip3gFg0ujgGvsxc/9OT39EgKiZoTv0QxxCCCGEEJIvLMs5DLEEed9AAn2eCH9zgMWovtqdkJsp2daK8DtL7IuIXXO/GJnehBBCCCFk+FAxgn/tzg7sDYiYW1jid9OuLjS399rLt7b2YMuebvt9c0cvNu7qyrkPSzbtSRuVWWL9vmXbcPfLTa51zR298CMWDX9bLGGfcDz/iQgQsQQ/Y/yEEEIIIcMGluXMwKW3vogbn1qXsU2/WSXnjBuewbyfPGkv/9a9K/CNu1+z3594/VM484Zncjr+a1tacfH8RVi8wV0JyKqh//DKHfjr4s328ngiiV2dmQcoY2pimD7efwZdiw+8bSYAYOyoKntZRARV5nEvedt+4U+CEEIIIYSUlKbWnoLvM5a9Sfmjqtjd2Y+mvZkvkF9ZTMDw0u/tTonvRB7ZElv2dtv7chL1mSXX6ktXXxwA8PhXTkNrzwAunr/I1ebl757jsur48ZWz5+DzZxzoSvKNiCAWjeD1a89LleckhBBCCCFlT3+AXh0MFSH4+xNJxJMa6Im3CCqL2dUfR3NHH1QVkk1hB9Dc3mfvy0nQ3pyCf/9J9ZgykN636lh2sS4iabPuirmZdzkhhBBCCBl5DCr8KyK/EJHXRWS5iPxbRMY71l0jIo0i8oaIvGPQPc1Ad58hllva/T3xFr0DSd/M566+OPrjSbT3uMV6ModIf0tnn70vJ0GDjN6BBLr6E6iKCqpjkbRE3cEQyXPQQgghhBBCKo/B+j0eB3CEqh4FYC2AawBARA4DcAmAwwGcB+B3IlK0cHOnKbJbOvt8Bb2VuNoXT7jq3VtYAwZvEm2PT9Q9CCvC393v3ibIRtQXT6K7L456c+KsXBJ1s5GhqA8hhBBCCBlhDEplqupjqmqFtBcDmGG+vgjAnarap6obATQCmDeYY2XCEtkDCUVr9wDW7uxAV18cr765F6qK/rgh+Jdu2otXNrfa263a1mZG2s0BQ0efq1rPK2/uxYJ1LeY+kli5tS3t2G3dA3hk5Q6sb+k0j7EHXX1xPLlmJzbv7nKV4nQG3pdu2oO2ngHUVxfeVcUIPyGEEEIIsSik2vwUgH+ar6fDGABYNJnL0hCRywBcBgD77ZdfRRmnb35raw/e+X8L7ff/96FjbFvNwsZdWNi4y1534Y0LcdHR+9ozmjV39OHDt7xor7/01pcAAI9eeRruXPIm/vT8Jjz7jTOw/6R6u81NzzTi5uc22O+ffqMFJ/30KbT1DOCAhnqMM6vn7DuuFucdMQ23Pb8RAPCNu5cDAOZMGe06l6Nnjs/rGjih3ieEkPw4dc5kLFi3K3tDQggZRmQV/CLyBICpPqu+rar3mW2+DSAO4O/WZj7tfQ3xqnozgJsBYO7cuXlVHnX65ldva3etW9fcmTbhlZMn1zTbr4Pq4m9t7caSTUa5zV2dfS7B37S3O619W88AAKPe/6FTx+LUOZNxy8fnoioSwbmH74NLbk6NhSxLDwCs+dF5GSfZCgsj/IQQkjt/+dQ8HDl9HI659vFSd4UQQgpKVsGvqmdnWi8iHwfwTgBv15SBvgnATEezGQC25dvJbHT1pXzzq7al226CEmcB99OBoLKeLY7qP7s9tfNbMlQGUhjJuWNHVaEmZqQwzJ5c72pTX5NKbRhVoMRdCn5CCMmdA6eMxoT66tDtX/rW2zFlbC3640lceOMCHLf/BNy5ZEsRe0gIGQlMzOF7KCyDrdJzHoBvAni3qjpD3fcDuEREakRkNoA5AF4azLEy4Yzwr/RE+Lv74oGJs4B7NrNVnm0trIRcAGmlPzOVAlUFtrf1osZRXnOS5ybWFcXDX/BdEkKILyJynlmNrVFErvZZLyJyo7l+uYgcG3bbocaa6PB77zzMtfywaWPxp0+8DWcc0oBnv3EGrn/fkXjLtLGYMrYWgFFC+fGvno7r33ck/vKpefjiWQfZy79+7sFDexKEkGFPMXTcYNXmbwHUAHjcrF+/WFUvV9VVInIXgNUwrD5fUNXwJW9ypNsRpfdG+Js7+tAbstqO39MBax9WcR+nwFdV12DAj56BhB3dB9Kr8YyuKbzgz3cuAUIIyQWz+tpNAM6B8WR3iYjcr6qrHc3OhxH0mQPgeAC/B3B8yG1LwqdOmY15sydi9fZ2XHX3clz7niNw3P4TcOahUwAYc6d8aF56zpmI4LSDG3DawQ342rmH2Ms/d/qBmPPthwEYOQJv7unG+4+dgQ++baY96/vtn5qH59a24NaFG/GOw/fBo6t24vvvOgwDiSQ+c8oB2NXVh827uzFzQh2eW9uCq+4x8sDW/Og8vLhxNz7xpyXFviyEkCGiGJOmDkptqupBGdZdB+C6wew/LF1mlZ6pY2uxw1OLf+3ODrT3xv02c+G3LQCMrY1h9fZ27DTXvbalFY+v3gnAmAktqHTnmNoYOszj1mSYQKuugPX3CSFkiJkHoFFVNwCAiNwJo0qbU7RfBOAvpuVzsYiMF5FpAGaF2LZgXPfeI/Dtf68M3f6I6eNwxPRx+MDcmdkbZ6EqGsGCq87EqOooJo+uca3b9NML7denH9yA73qeLlhMGVOLKWOMJwofeNtM/M9xM9AzkMCo6ijOOGQK3vjxeYhFIohGBC0dfaitimBMbRWeb9yFo2aMw++eWY8TD5iEqeNqce6vnrP3e98XTsbDK3dg/rPrMW/WRLxk5qtl4/wjpuLhlTtyvRRlx0NfOhUX3Lgga7uJ9dXY09WftZ0fDWNqMtp/C83HT9wfty/aXPTj/O/Fb0UiqXh9R4ddkKTcePdb90V9TRR3vDR4q90XzjwQNz29Hh+aNxNnHDIFX/nnMnT3J7DomrOwoaULD6/cjlgkgj+/sGnQx/rPFScPeh9eKmKm3a6+OCICHDptTJpof31HR6h9zJs9Efe/lp5mMHfWRDz1eiqx99m1LXh2bYu7zf4TsHTzXkweXYNd5gRcp86ZjEdW7kBSjT92J84//n3MR8KF4MIjp+HBFdsLtj9CCMnCdADOX9ImGFH8bG2mh9y2YHzk+P3xkeP3BwD8+fmN+MF/3eOKe/7fScU6NABg5sS6gu4vGhHXE2Lnk2Tnb87JB00GAHzzvEPtZc5BBgC8deZ4XH3+ofBj1bY2zJxYh7G1VVn7lEwq2noG7Hw052zvffEEVm5tQ39cMW/2RAwkkujuT9jn8OiqHbh14UZ8YO5MnHPYPphUX4323gGMG1WV8al170ACD63YjjMPmYLxdVVIKjIWv+joHUB1LIKICKrMKKr3emSjrWfArsDnPf/+RBKxiATOrdPdH0dbzwCmjRvlWm6lQOb6hF5VISLY29WPMbUx+7g/vOgI3/ZdfXFEI+K6N4ARwNzT1Y/Wnn5MHl2DSfXVdl92dfZhb1c/GsbUYHydv7f8e+/yH6gCwLbWHrR09CEigv0nG5+l9t4BjKmJQUTQO5CAiNGH5U1tmDFhFPafVI8Hlm/DuFFVaO+J48KjpqF3IJHWbydv7u7G9AmjfO//9e87CoBxj65/eA0uOno6/rpoM94ybQxOOHASDp06FgAQTyQRi0bQ2RdHTSyCqAgijv194x2pv5PVPzrPfj1t3Cj7b+0H7z4cvQMJqIbPzdzd2Yf23jj2dPXhuP0nhtomV8RvoqpSMXfuXF26dGnO2zV39GJnWx9mN9Rj064u7DO2Fru7+jB1bK2diDtzYp1dJyihxpfS2NoYdrb3YXRNDFPH1WLtzg5MqK9GbSyCvnjS/kLduKsLIsYN3dbqTuytjkVwUMNo7O3uR31NDH3xJLa19uCAhno0t/ehvXcAh+wzxvXH39NvTAC2rbUHc6aMLtikW9YXqN8XESFk+CMiL6vq3FL3w0JELgbwDlX9jPn+UgDzVPWLjjYPArheVRea758EcBWAA7Jt69iHs3zzcZs3Fy56qarY2z1QlCQ5QggZSjL9RlREhN/5qPOI6eMApCIcQaNR68t9kuPxqrWtF+fyoB8Faz+1VVFbcAdFdEZVRzEK0YIL86poBONGFd73RQghAYSpyBbUpjrEtgAKU745CBGh2CeEVDxUh4QQQvJlCYA5IjJbRKoBXAKjSpuT+wF8zKzWcwKANlXdHnJbQgghBaAiIvyEEEKGHlWNi8gVAB4FEAVwm1ml7XJz/XwADwG4AEAjgG4An8y0bQlOgxBCKh4KfkIIIXmjqg/BEPXOZfMdrxXAF8JuSwghpPDQ0kMIIYQQQkgFQ8FPCCGEEEJIBUPBTwghhBBCSAVDwU8IIYQQQkgFQ8FPCCGEEEJIBUPBTwghhBBCSAVDwU8IIYQQQkgFI0aJ5PJARFoAbM5z88kAdhWwO+XMSDlXnmflMVLOtVjnub+qNhRhv8MG/k7kxEg7X2DknTPPt/LJ5ZwDfyPKSvAPBhFZqqpzS92PoWCknCvPs/IYKec6Us5zuDHS7stIO19g5J0zz7fyKdQ509JDCCGEEEJIBUPBTwghhBBCSAVTSYL/5lJ3YAgZKefK86w8Rsq5jpTzHG6MtPsy0s4XGHnnzPOtfApyzhXj4SeEEEIIIYSkU0kRfkIIIYQQQoiHihD8InKeiLwhIo0icnWp+zMYROQ2EWkWkZWOZRNF5HERWWf+O8Gx7hrzvN8QkXeUpte5IyIzReRpEVkjIqtE5Mvm8ko811oReUlEXjPP9Yfm8oo7VwAQkaiIvCoiD5jvK+48RWSTiKwQkWUistRcVnHnOVzJ9psgBjea65eLyLGl6GehCHG+HzHPc7mIvCAiby1FPwtF2N98EXmbiCRE5P1D2b9iEOacReQM8ztplYg8O9R9LCQhPtPjROS/jt/VT5ain4XCT/t51g/+O0tVh/V/AKIA1gM4AEA1gNcAHFbqfg3ifE4DcCyAlY5lPwdwtfn6agA/M18fZp5vDYDZ5nWIlvocQp7nNADHmq/HAFhrnk8lnqsAGG2+rgLwIoATKvFczf5/FcA/ADxgvq+48wSwCcBkz7KKO8/h+F+Y3wQAFwB42PzbPAHAi6Xud5HP9yQAE8zX51f6+TraPQXgIQDvL3W/h+AejwewGsB+5vsppe53kc/3W47v2AYAewBUl7rvgzjnNO3nWT/o76xKiPDPA9CoqhtUtR/AnQAuKnGf8kZVn4PxwXVyEYDbzde3A3iPY/mdqtqnqhsBNMK4HmWPqm5X1VfM1x0A1gCYjso8V1XVTvNtlfmfogLPVURmALgQwC2OxRV3ngGMlPMsd8L8JlwE4C/m3+ZiAONFZNpQd7RAZD1fVX1BVfeabxcDmDHEfSwkYX/zvwjgHgDNQ9m5IhHmnD8M4F5VfRMAVHU4n3eY81UAY0REAIyGoZviQ9vNwhGg/ZwM+jurEgT/dABbHO+bzGWVxD6quh0whDKAKebyijh3EZkF4BgYke+KPFfT5rIMxo/P46paqef6awBXAUg6llXieSqAx0TkZRG5zFxWiec5HAlzvSvpnuR6Lp+GESkcrmQ9XxGZDuC9AOYPYb+KSZh7fDCACSLyjPm99LEh613hCXO+vwXwFgDbAKwA8GVVTaJyGfR3Vqyg3SkN4rNspJQeGvbnLiKjYURhrlTVdmOw7t/UZ9mwOVdVTQA4WkTGA/i3iByRofmwPFcReSeAZlV9WUTOCLOJz7KyP0+Tk1V1m4hMAfC4iLyeoe1wPs/hSJjrXUn3JPS5iMiZMAT/KUXtUXEJc76/BvBNVU1k+E0ZToQ55xiA4wC8HcAoAItEZLGqri1254pAmPN9B4BlAM4CcCCM7+EFqtpe5L6VikF/Z1WC4G8CMNPxfgaMEV8lsVNEpqnqdvMRjvWoblifu4hUwRD7f1fVe83FFXmuFqraKiLPADgPlXeuJwN4t4hcAKAWwFgR+Rsq7zyhqtvMf5tF5N8wHkFX3HkOU8Jc70q6J6HORUSOgmG1O19Vdw9R34pBmPOdC+BOU+xPBnCBiMRV9T9D0sPCE/YzvUtVuwB0ichzAN4KIz9uuBHmfD8J4KdqGNwbRWQjgEMBvDQ0XRxyBv2dVQmWniUA5ojIbBGpBnAJgPtL3KdCcz+Aj5uvPw7gPsfyS0SkRkRmA5iDYfJhN313twJYo6q/dKyqxHNtMCP7EJFRAM4G8Doq7FxV9RpVnaGqs2D8HT6lqh9FhZ2niNSLyBjrNYBzAaxEhZ3nMCbMb8L9AD5mVr44AUCbZccahmQ9XxHZD8C9AC4dphFfJ1nPV1Vnq+os87vobgCfH8ZiHwj3mb4PwKkiEhOROgDHw8iNG46EOd83YTzNgIjsA+AQABuGtJdDy6C/s4Z9hF9V4yJyBYBHYWR236aqq0rcrbwRkTsAnAFgsog0Afg+gJ8CuEtEPg3jQ34xAKjqKhG5C0ZmfhzAF0zryHDgZACXAlhhetsBI+u+Es91GoDbRSQKY5B9l6o+ICKLUHnn6kel3dN9YNiyAOM79B+q+oiILEFlneewJOg3QUQuN9fPh1G55QIYCdTdMKKFw5KQ5/s9AJMA/M783MZVdW6p+jwYQp5vRRHmnFV1jYg8AmA5jByqW1TVt8RjuRPyHl8L4M8isgKG3eWbqrqrZJ0eJAHarwoo3HcWZ9olhBBCCCGkgqkESw8hhBBCCCEkAAp+QgghhBBCKhgKfkIIIYQQQioYCn5CCCGEEEIqGAp+QgghhBBCSoiI3CYizSISqrqSiHxARFaLyCoR+UfW9qzSQwghhBBCSOkQkdMAdAL4i6oekaXtHAB3AThLVfeKyBRVbc60DSP8hBBCCCGElBBVfQ7AHucyETlQRB4RkZdFZIGIHGqu+iyAm1R1r7ltRrEPUPATQgghhBBSjtwM4IuqehyArwP4nbn8YAAHi8jzIrJYRM7LtqNhP9MuIYQQQgghlYSIjAZwEoB/mTNkA0CN+W8MwBwYs/POALBARI5Q1dag/VHwE0IIIYQQUl5EALSq6tE+65oALFbVAQAbReQNGAOAJZl2RgghhBBCCCkTVLUdhpi/GADE4K3m6v8AONNcPhmGxWdDpv1R8BNCCCGEEFJCROQOAIsAHCIiTSLyaQAfAfBpEXkNwCoAF5nNHwWwW0RWA3gawDdUdXfG/bMsJyGEEEIIIZULI/yEEEIIIYRUMBT8hBBCCCGEVDAU/IQQQgghhFQwFPyEEEIIIYRUMBT8hBBCCCGEVDAU/IQQQgghhFQwFPyEEEIIIYRUMBT8hBBCCCGEVDD/H2aBE1KFHjzqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_training(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()\n",
    "\n",
    "plot_training(i, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test  =[1,2,3]\n",
    "test[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
